kbforge-sql 0.1.0__py3-none-any.whl

kbforge_sql/__init__.py

@@ -0,0 +1,3 @@
+"""A relational database as a kbforge source, through configuration alone."""
+
+__version__ = "0.1.0"

kbforge_sql/config.py

@@ -0,0 +1,182 @@
+"""Config model for a SQL source, and the offline validation the CLI runs
+before any connection (spec §3.1). `check_columns` is the one check that needs
+the query result (§3.2)."""
+
+from __future__ import annotations
+
+import os
+import re
+from collections import Counter
+from collections.abc import Mapping, Sequence
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from kbforge.canonical import is_blank
+from kbforge.synthesize import OKF_OWNED
+from kbforge_sql.errors import SqlSourceError
+
+# Same shapes as kbforge-mcp's config: an ALL_CAPS env var name, and a `system`
+# that is safe inside a doc_id and a `sync/<system>` branch name.
+_ENV_NAME = re.compile(r"^[A-Z][A-Z0-9_]*\Z")
+_SYSTEM_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*\Z")
+
+
+class _Strict(BaseModel):
+    # A typo in a source config must be an error, not a silently ignored key.
+    model_config = ConfigDict(extra="forbid")
+
+
+class GroupSpec(_Strict):
+    children: list[str] = Field(min_length=1)
+    order_by: list[str] = Field(default_factory=list)
+    heading: str | None = None
+
+
+class SqlSourceConfig(_Strict):
+    system: str
+    url_env: str
+    password_env: str | None = None
+    query: str
+    id: list[str] = Field(min_length=1)
+    title: str
+    text: str | None = None
+    facets: list[str] = Field(default_factory=list)
+    exclude: list[str] = Field(default_factory=list)
+    type: str = "concept"
+    url_template: str | None = None
+    group: GroupSpec | None = None
+    retries: int = Field(default=2, ge=0)
+    max_removed_fraction: float = Field(default=0.5, gt=0, le=1)
+
+    def configured_columns(self) -> list[str]:
+        """Every column the config names, first mention first, no repeats."""
+        cols = [*self.id, self.title]
+        if self.text:
+            cols.append(self.text)
+        cols += [*self.facets, *self.exclude]
+        if self.group is not None:
+            cols += [*self.group.children, *self.group.order_by]
+        return list(dict.fromkeys(cols))
+
+
+# `url_template` is plain `{column}` substitution, deliberately not str.format:
+# format() would read `{a.b}` as an attribute, `{a[0]}` as an index, `{0}` as a
+# positional argument and `{a:{w}}` as a nested field, so a template could pass
+# validation and still fail mid-fetch. One pattern serves both sides.
+_TEMPLATE_FIELD = re.compile(r"\{([^{}]*)\}")
+
+
+def template_fields(template: str) -> list[str] | None:
+    """Column names a url_template references, or None if its braces don't
+    pair up into `{column}` fields."""
+    rest = _TEMPLATE_FIELD.sub("", template)
+    if "{" in rest or "}" in rest:
+        return None
+    return _TEMPLATE_FIELD.findall(template)
+
+
+def render_template(template: str, values: Mapping[str, str]) -> str:
+    return _TEMPLATE_FIELD.sub(lambda m: values[m.group(1)], template)
+
+
+def problems_for(config: dict) -> list[str]:
+    """Human-readable problems; `[]` means the config is usable. No connection."""
+    try:
+        cfg = SqlSourceConfig.model_validate(config)
+    except ValidationError as exc:
+        return [
+            f"config {'.'.join(str(p) for p in e['loc']) or '<root>'}: {e['msg']}"
+            for e in exc.errors()
+        ]
+
+    problems: list[str] = []
+    if not _SYSTEM_NAME.match(cfg.system):
+        problems.append(
+            "config 'system' must be a short identifier -- letters, digits, '_' "
+            "and '-', starting with a letter or digit -- because it is part of "
+            f"every doc_id and the publish branch: {cfg.system!r}"
+        )
+    for key, name in (("url_env", cfg.url_env), ("password_env", cfg.password_env)):
+        if name is None:
+            continue
+        if not _ENV_NAME.match(name):
+            # The value is NOT echoed: a name that fails this shape is most
+            # likely a pasted URL or password.
+            problems.append(
+                f"config '{key}' must name an environment variable (ALL_CAPS), "
+                "not hold its value"
+            )
+        elif name not in os.environ:
+            problems.append(f"config '{key}': environment variable {name} is not set")
+    if is_blank(cfg.query):
+        problems.append("config 'query' is blank")
+    if is_blank(cfg.type):
+        problems.append("config 'type' is blank; OKF requires a non-empty type")
+    if is_blank(cfg.title):
+        problems.append("config 'title' is blank")
+    if any(is_blank(c) for c in cfg.id):
+        problems.append("config 'id' has a blank column name")
+
+    used = {*cfg.id, cfg.title, *cfg.facets}
+    if cfg.text:
+        used.add(cfg.text)
+    if cfg.group is not None:
+        # Excluded columns are dropped from every row before grouping, so a
+        # child or order_by column in `exclude` would fail mid-fetch instead.
+        used |= {*cfg.group.children, *cfg.group.order_by}
+    if clash := used & set(cfg.exclude):
+        problems.append(
+            f"config 'exclude' names column(s) the source also uses: {sorted(clash)}"
+        )
+    if owned := sorted(set(cfg.facets) & OKF_OWNED):
+        problems.append(
+            f"config 'facets' must not use OKF-owned key(s) {owned}: the emitter "
+            "drops them, so the facet would silently never appear"
+        )
+
+    if cfg.group is not None:
+        children = set(cfg.group.children)
+        if ids := sorted(children & set(cfg.id)):
+            problems.append(
+                f"config 'group.children' must not include id column(s) {ids}"
+            )
+        entity = {cfg.title, *cfg.facets} | ({cfg.text} if cfg.text else set())
+        if both := sorted(children & entity):
+            problems.append(
+                "config 'group.children' must not include title, text or facet "
+                f"column(s) {both}: they describe the entity, not a child"
+            )
+        if extra := [c for c in cfg.group.order_by if c not in children]:
+            problems.append(
+                f"config 'group.order_by' must be a subset of 'group.children': {extra}"
+            )
+
+    if cfg.url_template is not None:
+        fields = template_fields(cfg.url_template)
+        if fields is None:
+            problems.append(
+                "config 'url_template' has unpaired braces; fields are `{column}`"
+            )
+        elif bad := sorted({f for f in fields if f not in cfg.id}):
+            problems.append(
+                f"config 'url_template' may only reference id column(s): {bad}"
+            )
+    return problems
+
+
+def check_columns(cfg: SqlSourceConfig, columns: Sequence[str]) -> None:
+    """Spec §3.2: every configured column must be in the result, and the
+    result must not name one column twice -- a duplicate makes 'which one did
+    the config mean' ambiguous, silently picking whichever `result.keys()`
+    binds a lookup to."""
+    missing = [c for c in cfg.configured_columns() if c not in columns]
+    if missing:
+        raise SqlSourceError(
+            f"configured column(s) {missing} are not in the query result; "
+            f"the query returned: {list(columns)}"
+        )
+    dupes = sorted(c for c, n in Counter(columns).items() if n > 1)
+    if dupes:
+        raise SqlSourceError(
+            f"the query returned duplicate column name(s) {dupes}; alias them apart"
+        )

kbforge_sql/connector.py

@@ -0,0 +1,385 @@
+"""The kbforge connector: four hookimpls over one rolled-back query.
+
+`kbforge_fetch` may use a clock and the network; `kbforge_normalize` may not
+(architecture §4.3). `retrieved_at` is stamped in fetch, into `anchor_hint`,
+and normalize only reads it back. Everything normalize needs travels in the
+record -- it never sees config."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from decimal import Decimal
+from urllib.parse import quote
+
+from sqlalchemy import URL, create_engine, make_url
+from sqlalchemy.exc import ArgumentError, DBAPIError, InterfaceError, OperationalError
+from sqlalchemy.pool import NullPool
+
+from kbforge.canonical import content_hash, is_blank
+from kbforge.hookspecs import hookimpl
+from kbforge.models import (
+    CanonicalDocument,
+    ConnectorInfo,
+    Cursor,
+    FetchResult,
+    RawRecord,
+    ResourceAnchor,
+)
+from kbforge_sql.config import (
+    SqlSourceConfig,
+    check_columns,
+    problems_for,
+    render_template,
+)
+from kbforge_sql.errors import SqlSourceError
+from kbforge_sql.identity import native_id_for
+from kbforge_sql.render import render_text
+from kbforge_sql.values import Scalar, canonical
+
+NAME = "sql"
+TOMBSTONE = "application/vnd.kbforge.tombstone"
+
+# Indirection so tests can observe backoff without waiting (Task 6).
+_sleep = time.sleep
+
+
+@dataclass(frozen=True)
+class _Entity:
+    native_id: str
+    payload: dict
+    url: str | None
+
+
+def _query_once(url: URL, query: str) -> tuple[list[str], list[tuple]]:
+    """Run the one configured statement and roll back. `exec_driver_sql`, not
+    `text()`: text() parses `:name` as a bind parameter, which breaks `'10:30'`
+    literals and Postgres `::` casts in an operator's query."""
+    engine = create_engine(url, poolclass=NullPool)
+    try:
+        with engine.connect() as conn:
+            try:
+                # no_parameters: a driver's paramstyle (psycopg/psycopg2/pymysql,
+                # and Denodo's dialect, are all pyformat-family) otherwise treats
+                # `%` in the statement as a parameter marker, and a bare `LIKE
+                # 'EV-%'` breaks with an empty parameter collection to fill it.
+                result = conn.exec_driver_sql(
+                    query, execution_options={"no_parameters": True}
+                )
+                if not result.returns_rows:
+                    raise SqlSourceError(
+                        "the query returned no result set; a source query must "
+                        "be a SELECT"
+                    )
+                columns = list(result.keys())
+                rows = [tuple(r) for r in result.fetchall()]
+            finally:
+                # No commit exists anywhere in this package. Whatever the
+                # statement did, this discards it (spec §7). SQLAlchemy also
+                # rolls back on close, so this line is belt-and-braces; no
+                # test can observe it, and the property tests pin is that
+                # nothing is ever committed.
+                conn.rollback()
+    finally:
+        engine.dispose()
+    return columns, rows
+
+
+# Retried: the connection failed, not the statement. Re-running a SELECT is
+# safe. Everything else -- bad SQL, a missing view, no permission -- fails at
+# once, because retrying a typo only delays the message. The classification is
+# the driver's: sqlite3 and pymysql raise OperationalError for some statement
+# errors too, so on those a bad query is retried before it fails. That costs
+# time, never correctness; `retries: 0` opts out.
+_TRANSIENT = (OperationalError, InterfaceError)
+
+
+def _url(cfg: SqlSourceConfig) -> URL:
+    try:
+        url = make_url(os.environ[cfg.url_env])
+    except ArgumentError:
+        # Never echo the value: it may carry a password.
+        raise SqlSourceError(
+            f"the value of {cfg.url_env} is not a SQLAlchemy URL"
+        ) from None
+    if cfg.password_env:
+        # URL.set takes the raw password: no percent-escaping for `@:/%`.
+        url = url.set(password=os.environ[cfg.password_env])
+    return url
+
+
+def _redact(message: str, url: URL) -> str:
+    secret = url.password
+    return message.replace(str(secret), "***") if secret else message
+
+
+def _query(cfg: SqlSourceConfig) -> tuple[list[str], list[tuple]]:
+    url = _url(cfg)
+    attempts = cfg.retries + 1
+    for attempt in range(1, attempts + 1):
+        try:
+            return _query_once(url, cfg.query)
+        except _TRANSIENT as exc:
+            if attempt == attempts:
+                raise SqlSourceError(
+                    _redact(
+                        f"{type(exc).__name__} after {attempts} attempt(s): {exc.orig}",
+                        url,
+                    )
+                ) from None
+            _sleep(min(2 ** (attempt - 1), 30))
+        except DBAPIError as exc:
+            raise SqlSourceError(
+                _redact(f"{type(exc).__name__}: {exc.orig}", url)
+            ) from None
+        except (ArgumentError, ImportError) as exc:
+            # NoSuchModuleError is an ArgumentError; a dialect whose DBAPI
+            # module is absent raises ImportError from create_engine.
+            raise SqlSourceError(
+                _redact(
+                    "no SQLAlchemy dialect or driver for this URL is installed "
+                    f"({exc}); install the driver for your database",
+                    url,
+                )
+            ) from None
+    raise AssertionError("unreachable: the loop returns or raises")
+
+
+def _sort_key(value: object) -> tuple:
+    """Order for a RAW database value in `group.order_by`. Raw, because the
+    canonical form is text for decimals and floats, and "10" < "9.5" as text.
+    Numbers compare by value across int/float/Decimal; other types by type name
+    first, so a column mixing types still sorts instead of raising; NULLs last."""
+    if value is None:
+        return (1,)
+    if isinstance(value, int | float | Decimal) and not isinstance(value, bool):
+        return (0, 0, Decimal(str(value)))
+    return (0, 1, type(value).__name__, value)
+
+
+def _entities(
+    cfg: SqlSourceConfig, columns: list[str], rows: list[tuple]
+) -> list[_Entity]:
+    index = {c: i for i, c in enumerate(columns)}
+    keep = [c for c in columns if c not in cfg.exclude]
+    children = list(cfg.group.children) if cfg.group else []
+
+    # Each canonical row keeps its raw tuple: group.order_by sorts on raw values.
+    by_id: dict[str, list[tuple[dict[str, Scalar], tuple]]] = {}
+    for raw in rows:
+        row = {c: canonical(raw[index[c]], c) for c in keep}
+        nid = native_id_for([row[c] for c in cfg.id], cfg.id)
+        by_id.setdefault(nid, []).append((row, raw))
+
+    entity_cols = [c for c in keep if c not in children]
+    entities: list[_Entity] = []
+    for nid in sorted(by_id):
+        group_rows = [row for row, _ in by_id[nid]]
+        if cfg.group is None and len(group_rows) > 1:
+            raise SqlSourceError(
+                f"{len(group_rows)} rows share the id {nid!r}; configure 'group' "
+                "to fold them, or make the id unique"
+            )
+        for c in entity_cols:
+            if len({json.dumps(r[c]) for r in group_rows}) > 1:
+                raise SqlSourceError(
+                    f"rows for id {nid!r} disagree on column {c!r}; only "
+                    "'group.children' columns may vary within an entity"
+                )
+        first = group_rows[0]
+
+        title = first[cfg.title]
+        lead = first[cfg.text] if cfg.text else None
+        attributes = [[c, first[c]] for c in cfg.id] + [
+            [c, first[c]]
+            for c in entity_cols
+            if c not in cfg.id and c != cfg.title and c != cfg.text
+        ]
+        group = None
+        if cfg.group is not None:
+            keyed = [
+                (
+                    [_sort_key(raw[index[c]]) for c in cfg.group.order_by],
+                    [row[c] for c in children],
+                )
+                for row, raw in by_id[nid]
+            ]
+            # A LEFT JOIN's all-NULL row means "no children", not a child.
+            keyed = [k for k in keyed if any(v is not None for v in k[1])]
+            keyed.sort(key=lambda k: (k[0], json.dumps(k[1])))
+            child_rows = [child for _, child in keyed]
+            group = {
+                "heading": cfg.group.heading or cfg.system,
+                "columns": children,
+                "rows": child_rows,
+            }
+        url = None
+        if cfg.url_template is not None:
+            url = render_template(
+                cfg.url_template, {c: quote(str(first[c]), safe="") for c in cfg.id}
+            )
+        entities.append(
+            _Entity(
+                native_id=nid,
+                url=url,
+                payload={
+                    "title": (
+                        nid if title is None or is_blank(str(title)) else str(title)
+                    ),
+                    "lead": None if lead is None else str(lead),
+                    "attributes": attributes,
+                    "facets": {f: first[f] for f in cfg.facets if first[f] is not None},
+                    "type": cfg.type,
+                    "group": group,
+                },
+            )
+        )
+    return entities
+
+
+_ALLOW_REMOVALS_ENV = "KBFORGE_SQL_ALLOW_REMOVALS"
+
+
+def _removals_allowed(system: str) -> bool:
+    """`KBFORGE_SQL_ALLOW_REMOVALS` is a comma-separated list of source
+    `system` names, out of band from the config on purpose:
+    `pipeline._instance_key` hashes the *whole* connector config into the
+    cursor slot name, so raising `max_removed_fraction` -- or editing any
+    other config key -- makes the next run find no prior manifest at all
+    (no tombstones, a silent NoOp) rather than performing the cleanup. This
+    is read here, at fetch time; `normalize` never reads the environment."""
+    raw = os.environ.get(_ALLOW_REMOVALS_ENV, "")
+    return system in {name.strip() for name in raw.split(",") if name.strip()}
+
+
+def _removed(cfg: SqlSourceConfig, prior: list[str], current: list[str]) -> list[str]:
+    """Ids seen at the last published run and missing now (spec §6).
+
+    The empty-result guard always applies. The deletion ceiling is skipped
+    when the source's `system` is listed in `KBFORGE_SQL_ALLOW_REMOVALS`,
+    the deliberate-cleanup override (spec §6.3). An empty result is refused
+    even at max_removed_fraction=1.0 or with the override set: a view
+    mid-refresh returns zero rows without an error, and 'delete the
+    knowledge base' must never be the default reading of that."""
+    if not prior:
+        return []
+    if not current:
+        raise SqlSourceError(
+            "the query returned no rows, but the last published run saw "
+            f"{len(prior)}; refusing to delete every concept. If the source "
+            "is meant to be empty, remove its config instead"
+        )
+    gone = sorted(set(prior) - set(current))
+    if _removals_allowed(cfg.system):
+        return gone
+    fraction = len(gone) / len(prior)
+    if fraction > cfg.max_removed_fraction:
+        raise SqlSourceError(
+            f"{len(gone)} of {len(prior)} previously seen ids ({fraction:.1%}) "
+            f"are missing, above max_removed_fraction={cfg.max_removed_fraction}; "
+            f"for a deliberate cleanup, rerun with {_ALLOW_REMOVALS_ENV}={cfg.system} "
+            "(editing the config resets deletion memory instead)"
+        )
+    return gone
+
+
+class SqlConnector:
+    @hookimpl
+    def kbforge_connector_info(self) -> ConnectorInfo:
+        return ConnectorInfo(
+            name=NAME,
+            version="0.1.0",
+            source_system="any database SQLAlchemy can reach",
+            info_types=["entity"],
+        )
+
+    @hookimpl
+    def kbforge_validate_config(self, config: dict) -> list[str]:
+        return problems_for(config)
+
+    @hookimpl
+    def kbforge_fetch(self, config: dict, cursor: Cursor | None) -> FetchResult:
+        cfg = SqlSourceConfig.model_validate(config)
+        try:
+            columns, rows = _query(cfg)
+            check_columns(cfg, columns)
+            entities = _entities(cfg, columns, rows)
+            current = [e.native_id for e in entities]
+            prior = list((cursor.payload.get("ids") if cursor else None) or [])
+            removed = _removed(cfg, prior, current)
+        except SqlSourceError as exc:
+            raise SqlSourceError(f"sql source {cfg.system!r}: {exc}") from None
+
+        stamped = datetime.now(tz=UTC).isoformat()
+
+        def hint(native_id: str, url: str | None) -> dict:
+            return {
+                "system": cfg.system,
+                "native_id": native_id,
+                "url": url,
+                "retrieved_at": stamped,
+            }
+
+        records = [
+            RawRecord(
+                anchor_hint=hint(e.native_id, e.url),
+                media_type="application/json",
+                payload=json.dumps(
+                    e.payload, sort_keys=True, ensure_ascii=False
+                ).encode(),
+            )
+            for e in entities
+        ]
+        records += [
+            RawRecord(anchor_hint=hint(nid, None), media_type=TOMBSTONE, payload=b"")
+            for nid in removed
+        ]
+        return FetchResult(
+            records=records,
+            cursor=Cursor(connector=NAME, payload={"ids": sorted(current)}),
+            complete=True,
+        )
+
+    @hookimpl
+    def kbforge_normalize(
+        self, records: Sequence[RawRecord]
+    ) -> list[CanonicalDocument]:
+        docs: list[CanonicalDocument] = []
+        for rec in records:
+            h = rec.anchor_hint
+            system, native_id = h["system"], h["native_id"]
+            anchor = ResourceAnchor(
+                system=system,
+                native_id=native_id,
+                url=h.get("url"),
+                retrieved_at=datetime.fromisoformat(h["retrieved_at"]),
+                content_hash="",
+            )
+            doc_id = f"{system}:{native_id}"
+            if rec.media_type == TOMBSTONE:
+                doc = CanonicalDocument(
+                    anchor=anchor,
+                    doc_id=doc_id,
+                    title=native_id,
+                    text="",
+                    deleted=True,
+                )
+            else:
+                p = json.loads(rec.payload)
+                doc = CanonicalDocument(
+                    anchor=anchor,
+                    doc_id=doc_id,
+                    title=p["title"],
+                    text=render_text(p["lead"], p["attributes"], p["group"]),
+                    structured={**p["facets"], "type": p["type"]},
+                )
+            doc.anchor.content_hash = content_hash(doc)
+            docs.append(doc)
+        return docs
+
+
+CONNECTOR = SqlConnector()

kbforge_sql/errors.py

@@ -0,0 +1,9 @@
+"""The one exception kbforge-sql raises."""
+
+
+class SqlSourceError(RuntimeError):
+    """A SQL source failed in a way an operator must fix or retry.
+
+    Raised unprefixed by the helpers; `kbforge_fetch` re-raises it once with
+    `sql source '<system>': ` in front, so every message names its source
+    exactly once."""

kbforge_sql/identity.py

@@ -0,0 +1,46 @@
+"""Id column values -> a path-safe, injective native_id (spec §4.2).
+
+The escape set is kbforge-mcp's (`kbforge_mcp/slug.py`, which documents each
+character). It is copied rather than imported: kbforge-sql must not depend on
+kbforge-mcp, and a shared helper belongs in core, which this release does not
+touch. Each id value is ONE path segment, so none of slug.py's URL and path
+handling applies -- only the per-segment escape and the two tail rules."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+
+from kbforge.canonical import is_blank
+from kbforge_sql.errors import SqlSourceError
+from kbforge_sql.values import Scalar
+
+# `%` makes the escape injective; `/` keeps a value from becoming structure;
+# the rest are illegal in an NTFS filename.
+_ESCAPE = re.compile(r'[\x00-\x1f\x7f%/<>:"|?*\\]')
+_UNSAFE_TAIL = (".", " ")
+
+
+def _escape(part: str) -> str:
+    escaped = _ESCAPE.sub(lambda m: f"%{ord(m.group()):02X}", part)
+    # Windows refuses, and sometimes silently trims, a trailing dot or space.
+    if escaped[-1:] in _UNSAFE_TAIL:
+        escaped = f"{escaped[:-1]}%{ord(escaped[-1]):02X}"
+    return escaped
+
+
+def native_id_for(values: Sequence[Scalar], columns: Sequence[str]) -> str:
+    parts: list[str] = []
+    for column, value in zip(columns, values, strict=True):
+        if value is None or (isinstance(value, str) and is_blank(value)):
+            raise SqlSourceError(
+                f"id column {column!r} is empty in a returned row; every row "
+                "needs an id"
+            )
+        parts.append(_escape(str(value)))
+    slug = "/".join(parts)
+    # `concept_path` strips a `.md` suffix a second time downstream, which would
+    # publish `x.md` and `x` to one file. Escaping the dot ends it.
+    if slug.endswith(".md"):
+        slug = f"{slug[:-3]}%2Emd"
+    return slug

kbforge_sql/render.py

@@ -0,0 +1,52 @@
+"""Canonical entity -> fixed-format markdown (spec §4.5).
+
+No templating: wording is the synthesizer's job. This only has to present the
+row faithfully and byte-stably, because this text is ALL a grounding reader
+sees of a row (`llm_synthesizer._grounding_block` passes `text`, not
+`structured`)."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from kbforge_sql.values import Scalar
+
+_NULL = "—"
+
+
+def _value(v: Scalar) -> str:
+    if v is None:
+        return _NULL
+    if isinstance(v, bool):
+        return "true" if v else "false"
+    return str(v).replace("\n", "<br>")
+
+
+def _cell(v: Scalar) -> str:
+    return _value(v).replace("|", "\\|")
+
+
+def render_text(
+    lead: str | None, attributes: Sequence[Sequence], group: dict | None
+) -> str:
+    blocks: list[str] = []
+    if lead:
+        blocks.append(lead)
+    blocks.append(
+        "\n".join(
+            [
+                "## Attributes",
+                *(f"- **{k}:** {_value(v)}" for k, v in attributes),
+            ]
+        )
+    )
+    if group is not None:
+        columns = group["columns"]
+        lines = [
+            f"## {group['heading']}",
+            "| " + " | ".join(_cell(c) for c in columns) + " |",
+            "|" + "---|" * len(columns),
+            *("| " + " | ".join(_cell(v) for v in row) + " |" for row in group["rows"]),
+        ]
+        blocks.append("\n".join(lines))
+    return "\n\n".join(blocks)

kbforge_sql/values.py

@@ -0,0 +1,54 @@
+"""Database value -> canonical JSON-safe form (spec §4.3).
+
+Everything the diff hashes passes through here, so this is where §4.3 law 1
+(determinism) is won or lost for a SQL source. Unknown types are rejected
+rather than str()-ed: a repr can embed memory addresses or driver-specific
+formatting, and a silently dropped column is a fact synthesis never sees."""
+
+from __future__ import annotations
+
+import math
+import unicodedata
+from datetime import UTC, date, datetime
+from decimal import Decimal
+from uuid import UUID
+
+from kbforge_sql.errors import SqlSourceError
+
+Scalar = str | int | bool | None
+
+
+def canonical(value: object, column: str) -> Scalar:
+    if value is None:
+        return None
+    # bool before int: bool is an int subclass, and True must not become 1.
+    if isinstance(value, bool | int):
+        return value
+    if isinstance(value, Decimal):
+        if not value.is_finite():
+            msg = f"column {column!r} holds a non-finite decimal {value}"
+            raise SqlSourceError(msg)
+        # normalize() drops trailing zeros (1.50 -> 1.5); format "f" undoes the
+        # exponent normalize() may introduce (100 -> 1E+2 -> "100").
+        return format(value.normalize(), "f")
+    if isinstance(value, float):
+        if not math.isfinite(value):
+            raise SqlSourceError(f"column {column!r} holds a non-finite float {value}")
+        return repr(value)
+    if isinstance(value, str):
+        text = value.replace("\r\n", "\n").replace("\r", "\n")
+        return unicodedata.normalize("NFC", text).rstrip()
+    # datetime before date: datetime is a date subclass.
+    if isinstance(value, datetime):
+        if value.utcoffset() is not None:
+            return value.astimezone(UTC).isoformat()
+        # Naive stays naive: guessing a timezone would invent a fact.
+        return value.isoformat()
+    if isinstance(value, date):
+        return value.isoformat()
+    if isinstance(value, UUID):
+        return str(value)
+    raise SqlSourceError(
+        f"column {column!r} holds a {type(value).__name__} value, which has no "
+        "canonical text form; add it to 'exclude' or cast it in the query"
+    )

kbforge_sql-0.1.0.dist-info/METADATA

@@ -0,0 +1,242 @@
+Metadata-Version: 2.5
+Name: kbforge-sql
+Version: 0.1.0
+Summary: SQL source connector for kbforge
+Author-email: Qing <qingye779@gmail.com>
+License: MIT
+Requires-Python: >=3.12
+Requires-Dist: kbforge>=0.8.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: sqlalchemy>=2.0
+Description-Content-Type: text/markdown
+
+# kbforge-sql
+
+`kbforge-sql` lets any database [SQLAlchemy](https://www.sqlalchemy.org/) can reach be a
+[kbforge](https://github.com/flyersworder/kbforge) source. Install it alongside kbforge and
+a source becomes one scoped `SELECT` instead of a Python package: each row the query
+returns (or each group of rows sharing an id) becomes one canonical document, rendered as
+fixed-format markdown. It registers itself under the `kbforge.connectors` entry-point
+group, so `kbforge list` shows `sql` with no further wiring. Every run is a full snapshot,
+which makes deletions derivable — an id seen last run and missing now becomes an explicit
+tombstone — so this is the first kbforge connector that emits tombstones at all.
+
+## Install
+
+```bash
+pip install kbforge-sql
+```
+
+`kbforge-sql` ships **no database driver**; the operator installs the one their engine
+needs, for example:
+
+```bash
+pip install denodo-sqlalchemy   # Denodo
+pip install "psycopg[binary]"   # PostgreSQL
+pip install oracledb            # Oracle
+pip install pyodbc              # SQL Server and other ODBC targets
+```
+
+## Configure a source
+
+A flat source — one row, one document:
+
+```yaml
+system: products                 # per-instance identity; prefixes every doc_id
+url_env: DENODO_URL              # env var NAME: denodo://kb_reader@host:9996/product_vdb
+password_env: DENODO_PASSWORD    # optional env var NAME; injected via URL.set()
+query: |
+  SELECT product_id, product_name, family, status, description, last_refreshed
+  FROM   iv_product
+  WHERE  app_id = 'EV-TRACTION'
+id: [product_id]                 # one or more columns -> native_id
+title: product_name
+text: description                # optional free-text column, placed verbatim
+facets: [family, status]         # scalar columns -> filterable frontmatter
+exclude: [last_refreshed]        # volatile columns, dropped before hashing
+type: product                    # OKF `type` for every concept; default "concept"
+url_template: https://portal.example/products/{product_id}   # optional; {column} fields, id columns only
+retries: 2                       # transient-error retries; default 2
+max_removed_fraction: 0.5        # deletion ceiling, see below; default 0.5
+```
+
+A grouped source — a joined view where one application spans many rows, folded into one
+document per application:
+
+```yaml
+system: applications
+url_env: DENODO_URL
+password_env: DENODO_PASSWORD
+query: |
+  SELECT app_id, app_name, segment, app_description,
+         product_id, product_name, product_status
+  FROM   iv_product_application
+  WHERE  app_id = 'EV-TRACTION'
+id: [app_id]
+title: app_name
+text: app_description
+facets: [segment]
+type: application
+group:
+  children: [product_id, product_name, product_status]
+  order_by: [product_id]
+  heading: Products              # optional; defaults to the source `system`
+```
+
+Columns outside `id` and `group.children` must be constant across an entity's rows; a
+grouped entity whose rows disagree on one is a fetch error naming the id and the column.
+
+kbforge takes connector config as repeated YAML-typed `--set` pairs, so that is one key
+per flag:
+
+```bash
+kbforge run --connector sql \
+  --set system=products \
+  --set url_env=DENODO_URL \
+  --set password_env=DENODO_PASSWORD \
+  --set 'query=SELECT product_id, product_name, family, status, description, last_refreshed FROM iv_product' \
+  --set 'id=[product_id]' \
+  --set title=product_name \
+  --set text=description \
+  --set 'facets=[family, status]' \
+  --set 'exclude=[last_refreshed]' \
+  --set type=product \
+  --mirror .kbforge/mirror --out .kbforge/out --state .kbforge/state
+```
+
+Because `--set` values are YAML-typed, a query with its own quoted string literal or a
+multi-line `WHERE` clause is easier to get right in a shell variable or script than typed
+inline; quote the whole `key=value` pair once and let the query itself carry its quotes.
+
+The query reaches the driver exactly as written, with no parameter binding, so `LIKE 'EV-%'`,
+Postgres `::` casts and `'10:30'` literals are safe on every driver. On the first result
+the connector checks that every configured column exists (listing the real ones if not)
+and that no two result columns share a name; `SELECT a.id, b.id` must alias them apart.
+
+## First run: `dry-run` and a throwaway mirror
+
+No tool proves a query executable across every dialect, so try a new source against a
+mirror you can throw away before pointing it at the real one. kbforge's default publisher
+is `dry-run`, so the query runs and every concept renders to local files with nothing
+opened anywhere:
+
+```bash
+kbforge run --connector sql --set … \
+  --mirror /tmp/try --state /tmp/try-state --out /tmp/try-out
+```
+
+Read the rendered files under `/tmp/try-out` before wiring in a real `--mirror` and
+`--publisher`.
+
+## Credentials
+
+`url_env` and `password_env` hold environment variable **names**, never values. A
+credential never appears in config, on a command line, or in an error message — errors
+name the env var, not its content. With `password_env` set, the URL carries no password
+and the raw value is injected with SQLAlchemy's `URL.set(password=...)`, so a password
+containing `@`, `:`, `/` or `%` needs no percent-escaping.
+
+The expected deployment is a service account, and the recommended one is a **dedicated
+read-only account** granted `SELECT` only on the views its queries use. The connector
+rolls back every transaction and never commits — no commit exists anywhere in the
+package — but that is a bound on accidents, not a guarantee: kbforge cannot prove a SQL
+string is free of side effects through a function call, a procedure, or a dialect
+extension, so the database's grants are what actually prevent a write. (The connector
+issues an explicit rollback even though SQLAlchemy also rolls back on connection close;
+belt-and-braces, since the account's grants are what really carry the guarantee.)
+
+`kbforge_validate_config` also rejects a blank `title` or a blank `id` column name before
+any connection is attempted, alongside the checks in §3.1 of the design note.
+
+A dropped or refused connection is retried up to `retries` times (default 2) with
+exponential backoff capped at 30 seconds; bad SQL, a missing view, or a missing grant fails
+at once, because retrying a typo only delays the message — on drivers that classify it that
+way (see "Known limits").
+
+## Deletions
+
+Every run is a full snapshot: the connector keeps the id set from the last published run
+in the cursor manifest, and any id missing from this run's result becomes an explicit
+tombstone. Two guards sit in front of that, because a view that returns nothing — a cache
+mid-refresh, a failed upstream load, a filter changed upstream — is not an error to the
+database:
+
+- **Empty result with a non-empty prior manifest fails the run** and emits no tombstones,
+  rather than reading "the source is empty" as "delete every concept". An intentionally
+  empty source is rare enough to be handled by removing its config instead.
+- **Deletion ceiling.** If the tombstones would exceed `max_removed_fraction` (default
+  `0.5`) of the prior manifest, the run fails and states the count and the fraction. For a
+  deliberate large cleanup, rerun with `KBFORGE_SQL_ALLOW_REMOVALS` set (see below) rather
+  than raising `max_removed_fraction` — see "Known limits" for why.
+
+## Known limits
+
+**Whether an error is retried is the driver's call.** The connector retries SQLAlchemy's
+`OperationalError` and `InterfaceError`, which most drivers reserve for connection
+failures. Some also raise `OperationalError` for statement errors — sqlite3 for a missing
+table or a syntax error, pymysql for access denied and unmapped server errors — so on those
+drivers a bad query is retried before it fails, and the message reads "OperationalError
+after N attempt(s)". That costs time, never correctness. PostgreSQL (psycopg) reports these
+as `ProgrammingError` and fails at once; check your driver with a deliberately misspelled
+view on a first run, and set `retries: 0` if it misclassifies.
+
+**Editing ANY config key resets deletion memory, not just the query.** Cursor slots are
+keyed by a digest of the *whole* connector config (`pipeline._instance_key`), so editing
+`query` — narrowing its `WHERE`, say — or any other key, including `max_removed_fraction`
+itself, means the next run finds no prior cursor: `NoOp`, no tombstones, and the old slot
+trips the ceiling again if the edit is ever reverted. The connector cannot fix this; it is
+deliberately mirror-blind. After narrowing a query, remove the stale concepts by hand in
+the review repository.
+
+**A tripped deletion ceiling is cleared with `KBFORGE_SQL_ALLOW_REMOVALS`, not by raising
+`max_removed_fraction`.** Raising the fraction is a config edit, so it hits the limit
+above: it resets deletion memory instead of performing the cleanup, and the old cursor
+slot trips the ceiling again once the fraction is reverted. Instead, set the environment
+variable `KBFORGE_SQL_ALLOW_REMOVALS` to a comma-separated list of source `system` names
+(for example `KBFORGE_SQL_ALLOW_REMOVALS=products`) and rerun with the config unchanged —
+it is read at fetch time, out of band from the config, so the cursor slot and the rest of
+`max_removed_fraction`'s guard stay intact. The empty-result guard always applies, even
+with the override set.
+
+**Ids can collide with another source's.** `concept_path` drops the system prefix, so a
+product id `42` and an application id `42` render the same file, and the pipeline aborts
+on the collision — numeric keys make this likely. Until bundle paths are system-qualified,
+give ids a kind prefix in the query itself (`'product-' || product_id AS kb_id`, or the
+`CONCAT` your dialect prefers) and use that column as `id`. No connector config is needed
+for this.
+
+**Ids differing only in case are one directory on a case-insensitive checkout.** `Foo` and
+`foo` render distinct `native_id`s but the same path on the default macOS or Windows
+filesystem, so a case-only id pair collides in a local checkout even though it would not
+on Linux CI.
+
+**Canonicalization can fold two raw ids into one entity.** NFC normalization and
+trailing-whitespace stripping (§4.3) run before the duplicate-id check, so two raw id
+values that canonicalize to the same string collapse onto one id: without `group`, that is
+the duplicate-id error; with `group`, it is a silent merge into one entity's rows.
+
+## Testing against your own database
+
+The package ships a live test that is skipped unless you ask for it. Point it at a
+read-only view and run it with your driver installed:
+
+```bash
+pip install kbforge-sql denodo-sqlalchemy pytest   # or psycopg[binary], oracledb, ...
+export KBFORGE_SQL_LIVE_URL='denodo://kb_reader@denodo.example:9996/product_vdb'
+export KBFORGE_SQL_LIVE_PASSWORD=...               # optional
+export KBFORGE_SQL_LIVE_QUERY="SELECT product_id, product_name FROM iv_product WHERE product_name LIKE '%a%'"
+export KBFORGE_SQL_LIVE_ID=product_id KBFORGE_SQL_LIVE_TITLE=product_name
+pytest packages/kbforge-sql/tests/test_sql_live.py --run-live
+```
+
+It fetches twice and requires identical content hashes, which is the check that matters
+for a new source: anything volatile in the result (a refresh timestamp, a computed
+column) fails it, and belongs in `exclude`. Without the `KBFORGE_SQL_LIVE_QUERY` variables
+it queries `information_schema`, which any PostgreSQL accepts.
+
+## Design
+
+The [design note](https://github.com/flyersworder/kbforge/blob/main/docs/design/2026-09-18-sql-source-connector-design.md)
+holds the full rationale and what remains deferred (incremental fetch, relations between
+rows, deletion memory across a query edit). The shipped design is in
+[`docs/architecture.md`](https://github.com/flyersworder/kbforge/blob/main/docs/architecture.md) §4.1.

kbforge_sql-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+kbforge_sql/__init__.py,sha256=ypEAJs517pwYG0Ujisr_yvzyK0IrpQ0w3bTELGql6mI,101
+kbforge_sql/config.py,sha256=0dNd7WG07b-oMJ-an9URI7Xwn3UoTZXca_-Iql1Y0Fw,7221
+kbforge_sql/connector.py,sha256=gyNJWuudZrwjRDHbusYm97dSI6tK9DsXmmXhAmRTeHA,14805
+kbforge_sql/errors.py,sha256=IZ8l9-8AuceTj28MrdtcSlW8N6h_e6giZ9qWML3oULE,322
+kbforge_sql/identity.py,sha256=xmMsvRSDiCMl0MznPu3iaGrmbWg5TWPpQfMT0wEt7-c,1850
+kbforge_sql/render.py,sha256=t23kwWBFyFMw8gKuVwAioJURouqDDcvX4DPmrwFOLt8,1427
+kbforge_sql/values.py,sha256=grkugIo7c-WQd9BUSpgFo5nTZDWJdTU54tZucd5CZCA,2152
+kbforge_sql-0.1.0.dist-info/METADATA,sha256=Wa8Ge_xyF5yOq97pd_LKQW2g0BSRF6bHIzEty3un-KY,12113
+kbforge_sql-0.1.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
+kbforge_sql-0.1.0.dist-info/entry_points.txt,sha256=GIXw1n48voMiVEYEEArEs8hWAlaMgTub06PlOsNihBM,59
+kbforge_sql-0.1.0.dist-info/RECORD,,

kbforge_sql-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.3
+Root-Is-Purelib: true
+Tag: py3-none-any

kbforge_sql-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[kbforge.connectors]
+sql = kbforge_sql.connector:CONNECTOR