dataface 0.1.2__py3-none-any.whl

dataface/core/inspect/search.py

@@ -0,0 +1,461 @@
+"""Full-text + filter search over the resolver's materialized schema search index.
+
+Sibling to ``LayeredSchemaResolver``: not a ``SchemaSource`` Protocol
+method. Find/filter is a different access pattern from drill-down.
+
+Substring + regex match across text-bearing fields, layered on top of
+structured predicate filters. Stable iteration order — no ranking, no
+scoring, no top-N. The schema search index is materialized fresh each
+call through the resolver; the resolver's own per-source caching keeps
+it cheap.
+
+Provenance flows through unchanged from the resolver: a hit on a name
+that came from the adapter carries ``source='dbt_adapter'``; a hit on a
+description that came from the cache carries ``source='super_schema'``;
+a hit on a tag/test/relationships from the manifest carries
+``source='dbt_manifest'``.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel, Field
+
+if TYPE_CHECKING:
+    from dataface.core.inspect.resolver import LayeredSchemaResolver
+
+
+_SNIPPET_INLINE_LIMIT: int = 200
+_SNIPPET_WINDOW: int = 80
+
+
+class InvalidRegex(ValueError):
+    """Raised when ``regex=True`` is paired with a non-compilable pattern."""
+
+
+class SchemaSearchHit(BaseModel):
+    """One match emitted by ``SchemaSearch.search``.
+
+    ``location`` is the dot-joined path: ``"<source>.<schema>.<table>"`` for a
+    table hit, ``"<source>.<schema>.<table>.<column>"`` for a column hit.
+    ``matched_field`` says which text-bearing field matched ("name",
+    "description", "tag", "tests.<name>", "meta.<key>", "owner", or
+    "_missing:<field>" for synthetic missing-field hits). ``source`` is
+    provenance — which layer contributed the field ("super_schema",
+    "dbt_manifest", "dbt_adapter").
+    """
+
+    location: str
+    matched_field: str
+    snippet: str
+    source: str
+    value: str
+    attrs: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Structured metadata attributes (role, tag, fk_to, etc.).",
+    )
+
+
+@dataclass
+class _SchemaSearchEntry:
+    """One table or column row, with its searchable fields and structured attrs."""
+
+    location: str
+    is_column: bool
+    consulted: tuple[str, ...]
+    text_fields: list[tuple[str, str]]  # (matched_field, value) — order-stable
+    attrs: dict[str, Any]  # structured filter inputs (role, tags, tests, ...)
+
+
+class SchemaSearch:
+    """Full-text + filter search over ``LayeredSchemaResolver``."""
+
+    def __init__(self, resolver: LayeredSchemaResolver) -> None:
+        self.resolver = resolver
+
+    def search(
+        self,
+        keyword: str,
+        *,
+        scope: list[str] | None = None,
+        regex: bool = False,
+        role: str | None = None,
+        tag: str | None = None,
+        has_test: str | None = None,
+        missing: str | None = None,
+        meta: dict[str, str] | None = None,
+        column_name: str | None = None,
+        table_name: str | None = None,
+        fk_to: str | None = None,
+    ) -> tuple[list[SchemaSearchHit], list[str]]:
+        pattern = self._compile_keyword(keyword, regex=regex)
+        hits: list[SchemaSearchHit] = []
+        entries, warnings = self._materialize_index()
+        for entry in entries:
+            if not self._matches_filters(
+                entry,
+                role=role,
+                tag=tag,
+                has_test=has_test,
+                meta=meta,
+                column_name=column_name,
+                table_name=table_name,
+                fk_to=fk_to,
+            ):
+                continue
+            if missing is not None:
+                hits.extend(self._missing_hits(entry, missing))
+                continue
+            if not keyword and any(
+                f is not None
+                for f in (role, tag, has_test, meta, column_name, table_name, fk_to)
+            ):
+                # Empty keyword + structured filter = enumerate qualifying entries
+                # without text matching. One hit per scope-eligible field.
+                hits.extend(self._unfiltered_hits(entry, scope))
+                continue
+            hits.extend(self._keyword_hits(entry, pattern, scope))
+        return hits, warnings
+
+    # ---- Schema index walk --------------------------------------------------
+
+    def _materialize_index(self) -> tuple[list[_SchemaSearchEntry], list[str]]:
+        """Walk source → schema → table → column. Stable order throughout.
+
+        Per-source / per-schema failures (unreachable warehouse, missing dbt
+        adapter package, etc.) are turned into ``warnings`` strings so the
+        verb wrapper can surface them — silent skips would let the user mistake
+        an incomplete schema walk for "no matches".
+
+        Calls ``profile_table`` per-table so each entry carries its own
+        ``sources_consulted`` — required for accurate per-field provenance
+        when a schema has a mix of cached and uncached tables.
+        """
+        entries: list[_SchemaSearchEntry] = []
+        warnings: list[str] = []
+        for source_entry in self.resolver.adapter_registry.list_sql_sources():
+            source_name = source_entry["name"]
+            try:
+                schemas_resp = self.resolver.list_schemas(source_name)
+            except Exception as exc:  # noqa: BLE001 — surface as warning
+                warnings.append(f"source '{source_name}': {exc}")
+                continue
+            schemas = (
+                schemas_resp.get("sources", {}).get(source_name, {}).get("schemas", {})
+            )
+            for schema_name in schemas:
+                try:
+                    tables_resp = self.resolver.list_tables(source_name, schema_name)
+                except Exception as exc:  # noqa: BLE001 — surface as warning
+                    warnings.append(
+                        f"source '{source_name}' schema '{schema_name}': {exc}"
+                    )
+                    continue
+                table_names = (
+                    tables_resp.get("sources", {})
+                    .get(source_name, {})
+                    .get("schemas", {})
+                    .get(schema_name, {})
+                    .get("tables", {})
+                )
+                for table_name in table_names:
+                    try:
+                        profile_resp = self.resolver.profile_table(
+                            source=source_name, schema=schema_name, table=table_name
+                        )
+                    except Exception as exc:  # noqa: BLE001 — surface as warning
+                        warnings.append(
+                            f"source '{source_name}' schema '{schema_name}' "
+                            f"table '{table_name}': {exc}"
+                        )
+                        continue
+                    consulted = tuple(
+                        profile_resp.get("_meta", {}).get("sources_consulted", [])
+                    )
+                    table_profile = (
+                        profile_resp.get("sources", {})
+                        .get(source_name, {})
+                        .get("schemas", {})
+                        .get(schema_name, {})
+                        .get("tables", {})
+                        .get(table_name, {})
+                    )
+                    entries.append(
+                        self._table_entry(
+                            source_name,
+                            schema_name,
+                            table_name,
+                            table_profile,
+                            consulted,
+                        )
+                    )
+                    for col_name, col_profile in (
+                        table_profile.get("columns") or {}
+                    ).items():
+                        entries.append(
+                            self._column_entry(
+                                source_name,
+                                schema_name,
+                                table_name,
+                                col_name,
+                                col_profile,
+                                consulted,
+                            )
+                        )
+        return entries, warnings
+
+    @staticmethod
+    def _table_entry(
+        source: str,
+        schema: str,
+        table: str,
+        profile: dict[str, Any],
+        consulted: tuple[str, ...],
+    ) -> _SchemaSearchEntry:
+        loc = f"{source}.{schema}.{table}"
+        text_fields: list[tuple[str, str]] = [("name", table)]
+        desc = profile.get("description")
+        if isinstance(desc, str) and desc:
+            text_fields.append(("description", desc))
+        owner = profile.get("owner")
+        if isinstance(owner, str) and owner:
+            text_fields.append(("owner", owner))
+        for t in profile.get("tags") or []:
+            if isinstance(t, str) and t:
+                text_fields.append(("tag", t))
+        attrs: dict[str, Any] = {}
+        for k in ("tags", "owner", "row_count", "kind", "grain"):
+            if k in profile:
+                attrs[k] = profile[k]
+        return _SchemaSearchEntry(
+            location=loc,
+            is_column=False,
+            consulted=consulted,
+            text_fields=text_fields,
+            attrs=attrs,
+        )
+
+    @staticmethod
+    def _column_entry(
+        source: str,
+        schema: str,
+        table: str,
+        column: str,
+        profile: dict[str, Any],
+        consulted: tuple[str, ...],
+    ) -> _SchemaSearchEntry:
+        loc = f"{source}.{schema}.{table}.{column}"
+        text_fields: list[tuple[str, str]] = [("name", column)]
+        desc = profile.get("description")
+        if isinstance(desc, str) and desc:
+            text_fields.append(("description", desc))
+        for t in profile.get("tags") or []:
+            if isinstance(t, str) and t:
+                text_fields.append(("tag", t))
+        for test in profile.get("tests") or []:
+            tname = test.get("name") if isinstance(test, dict) else None
+            if isinstance(tname, str) and tname:
+                text_fields.append((f"tests.{tname}", tname))
+        meta = profile.get("meta")
+        if isinstance(meta, dict):
+            for k, v in meta.items():
+                if isinstance(v, str) and v:
+                    text_fields.append((f"meta.{k}", v))
+        attrs: dict[str, Any] = {}
+        for k in (
+            "role",
+            "key_role",
+            "semantic_type",
+            "tags",
+            "tests",
+            "relationships",
+            "meta",
+            "type",
+            "actual_type",
+            "declared_type",
+        ):
+            if k in profile:
+                attrs[k] = profile[k]
+        return _SchemaSearchEntry(
+            location=loc,
+            is_column=True,
+            consulted=consulted,
+            text_fields=text_fields,
+            attrs=attrs,
+        )
+
+    # ---- Match helpers ------------------------------------------------------
+
+    @staticmethod
+    def _compile_keyword(keyword: str, regex: bool) -> re.Pattern[str] | None:
+        if not keyword:
+            return None
+        if regex:
+            try:
+                return re.compile(keyword, re.IGNORECASE)
+            except re.error as exc:
+                raise InvalidRegex(f"invalid regex: {exc}") from exc
+        return re.compile(re.escape(keyword), re.IGNORECASE)
+
+    def _matches_filters(
+        self,
+        entry: _SchemaSearchEntry,
+        role: str | None,
+        tag: str | None,
+        has_test: str | None,
+        meta: dict[str, str] | None,
+        column_name: str | None,
+        table_name: str | None,
+        fk_to: str | None,
+    ) -> bool:
+        if role is not None:
+            if not entry.is_column or entry.attrs.get("role") != role:
+                return False
+        if tag is not None:
+            tags = entry.attrs.get("tags") or []
+            if tag not in tags:
+                return False
+        if has_test is not None:
+            if has_test == "relationships":
+                if not entry.attrs.get("relationships"):
+                    return False
+            else:
+                tests = entry.attrs.get("tests") or []
+                if not any(
+                    isinstance(t, dict) and t.get("name") == has_test for t in tests
+                ):
+                    return False
+        if fk_to is not None:
+            rels = entry.attrs.get("relationships") or []
+            if not any(
+                isinstance(r, dict) and r.get("to_table") == fk_to for r in rels
+            ):
+                return False
+        if column_name is not None:
+            if not entry.is_column:
+                return False
+            cname = entry.location.rsplit(".", 1)[-1]
+            if not fnmatch.fnmatch(cname, column_name):
+                return False
+        if table_name is not None:
+            parts = entry.location.split(".")
+            # source.schema.table[.column] — table is index 2.
+            if len(parts) < 3 or not fnmatch.fnmatch(parts[2], table_name):
+                return False
+        if meta is not None:
+            entry_meta = entry.attrs.get("meta") or {}
+            if not isinstance(entry_meta, dict):
+                return False
+            for k, v in meta.items():
+                if str(entry_meta.get(k)) != v:
+                    return False
+        return True
+
+    def _missing_hits(
+        self, entry: _SchemaSearchEntry, field_name: str
+    ) -> list[SchemaSearchHit]:
+        present = any(name == field_name for name, _ in entry.text_fields)
+        if present:
+            return []
+        return [
+            SchemaSearchHit(
+                location=entry.location,
+                matched_field=f"_missing:{field_name}",
+                snippet="",
+                source=self._provenance_for(entry, field_name),
+                value="",
+                attrs=dict(entry.attrs),
+            )
+        ]
+
+    def _unfiltered_hits(
+        self, entry: _SchemaSearchEntry, scope: list[str] | None
+    ) -> list[SchemaSearchHit]:
+        hits: list[SchemaSearchHit] = []
+        for matched_field, value in entry.text_fields:
+            if scope and not _scope_matches(matched_field, scope):
+                continue
+            hits.append(
+                SchemaSearchHit(
+                    location=entry.location,
+                    matched_field=matched_field,
+                    snippet=_snippet(value, None),
+                    source=self._provenance_for(entry, matched_field),
+                    value=value,
+                    attrs=dict(entry.attrs),
+                )
+            )
+        return hits
+
+    def _keyword_hits(
+        self,
+        entry: _SchemaSearchEntry,
+        pattern: re.Pattern[str] | None,
+        scope: list[str] | None,
+    ) -> list[SchemaSearchHit]:
+        if pattern is None:
+            return []
+        hits: list[SchemaSearchHit] = []
+        for matched_field, value in entry.text_fields:
+            if scope and not _scope_matches(matched_field, scope):
+                continue
+            m = pattern.search(value)
+            if m is None:
+                continue
+            hits.append(
+                SchemaSearchHit(
+                    location=entry.location,
+                    matched_field=matched_field,
+                    snippet=_snippet(value, m),
+                    source=self._provenance_for(entry, matched_field),
+                    value=value,
+                    attrs=dict(entry.attrs),
+                )
+            )
+        return hits
+
+    @staticmethod
+    def _provenance_for(entry: _SchemaSearchEntry, matched_field: str) -> str:
+        """Attribute a hit to a single layer using the resolver's consulted set.
+
+        Cache-only path: everything came from super_schema. Otherwise we
+        classify the matched field — manifest-only fields (description,
+        tag, tests, owner, meta, relationships, declared_type) attribute
+        to ``dbt_manifest`` when the manifest was consulted; everything
+        else (names, types) attributes to ``dbt_adapter``.
+        """
+        if entry.consulted == ("super_schema",):
+            return "super_schema"
+        manifest_field = matched_field in {"description", "tag", "owner"} or any(
+            matched_field.startswith(p) for p in ("tests.", "meta.")
+        )
+        if manifest_field and "dbt_manifest" in entry.consulted:
+            return "dbt_manifest"
+        return "dbt_adapter"
+
+
+# ---- Module helpers --------------------------------------------------------
+
+
+def _scope_matches(matched_field: str, scope: list[str]) -> bool:
+    """Match an emitted matched_field against a list of scope prefixes."""
+    head = matched_field.split(".", 1)[0]
+    return head in scope or matched_field in scope
+
+
+def _snippet(value: str, match: re.Match[str] | None) -> str:
+    """Whole-field snippet when short; an 80-char window centered on first match."""
+    if len(value) < _SNIPPET_INLINE_LIMIT:
+        return value
+    if match is None:
+        return value[:_SNIPPET_WINDOW]
+    span = match.start()
+    half = _SNIPPET_WINDOW // 2
+    start = max(0, span - half)
+    end = min(len(value), start + _SNIPPET_WINDOW)
+    start = max(0, end - _SNIPPET_WINDOW)
+    return value[start:end]

dataface/core/inspect/sources/__init__.py

@@ -0,0 +1,32 @@
+"""Plugin-boundary schema sources.
+
+Schema sources are horizontal contributors of facts about a warehouse
+schema. Implementations are independently testable; the resolver composes
+contributions across sources into the wire response.
+
+Two peers ship today:
+
+  * `SuperSchemaSource` — warm cache backed by ``target/super_schema.json``.
+    Surfaces the rich profile (stats, distributions, semantic types, role
+    tags, enum lists) that a profiling run computed.
+  * `DbtSchemaSource` — cold-start, composes a dbt-core adapter + lazy
+    ``target/manifest.json`` reader. Surfaces only what's directly
+    available — no inference, no naming heuristics. Honest empty.
+
+Each source exposes ``name``, ``generated_at``, ``list_schemas``,
+``list_tables``, ``profile_table``, and ``describe_query``. Methods return
+bare-scalar ``dict | None`` (no per-field provenance wrapping; provenance
+lives only in the response-level ``_meta`` footer).
+``None`` means "this source has nothing to contribute" — silent
+no-contribution, not an error. The resolver falls through to the next
+source. Field-name conventions communicate origin: ``actual_type`` came
+from the adapter; ``declared_type`` / ``description`` / ``tests`` /
+``relationships`` came from the manifest. The resolver stamps a single
+response-level ``_meta`` footer with ``retrieved_at``, ``sources_consulted``,
+and (when super_schema contributed) ``cache_built_at``.
+
+Each source is *per-source* by construction: ``SuperSchemaSource`` is one
+instance per project (its cache file is per-project); ``DbtSchemaSource``
+is one instance per warehouse adapter. The resolver routes calls to the
+right instance, so the methods don't take a ``source`` argument.
+"""