classifyre-cli 0.7.0__tar.gz → 0.7.2__tar.gz

classifyre_cli-0.7.2/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 325 packages in 204ms
+Checked 52 packages in 4ms

{classifyre_cli-0.7.0 → classifyre_cli-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.7.0
+Version: 0.7.2
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.7.0 → classifyre_cli-0.7.2}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.7.0",
+  "version": "0.7.2",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.7.0 → classifyre_cli-0.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.7.0"
+version = "0.7.2"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably

{classifyre_cli-0.7.0 → classifyre_cli-0.7.2}/src/graph/edges.py

@@ -131,6 +131,7 @@ class FieldTransform(StrEnum):
     IDENTITY = "IDENTITY"  # copied through unchanged
     TRANSFORMED = "TRANSFORMED"  # an expression combined the upstreams
     AGGREGATED = "AGGREGATED"  # SUM/COUNT/... over a group
+    MAPPED = "MAPPED"  # the platform says it feeds this column, not how
     INDIRECT = "INDIRECT"  # influenced the rows, not the values
 
 

{classifyre_cli-0.7.0 → classifyre_cli-0.7.2}/src/main.py

@@ -971,6 +971,14 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                             source.partial_coverage_reason or "no reason given",
                         )
 
+                    if getattr(source, "listing_complete", False) and hasattr(
+                        sink, "set_listing_complete"
+                    ):
+                        sink.set_listing_complete()
+
+                    if hasattr(sink, "set_sweep_complete"):
+                        sink.set_sweep_complete(getattr(source, "sweep_complete", None))
+
                     # Relationships go *before* finish(): finish() finalizes the
                     # run and marks it COMPLETED, and edges that land after that
                     # belong to a run that already claimed to be done.

{classifyre_cli-0.7.0 → classifyre_cli-0.7.2}/src/outputs/rest.py

@@ -203,6 +203,10 @@ class FinalizeIngestRunRequest(BaseModel):
     # "This run covered a slice of the source." Suppresses retirement: absence
     # from a cohort run is not evidence that an asset is gone.
     partial_coverage: bool | None = Field(None, serialization_alias="partialCoverage")
+    discovery_complete: bool | None = Field(None, serialization_alias="discoveryComplete")
+    # Whether every unit of an AUTOMATIC sweep that wraps has been covered once.
+    # Only sent by a run with a wrapping unit; see BaseSource.sweep_complete.
+    sweep_complete: bool | None = Field(None, serialization_alias="sweepComplete")
     # Per ctx.cohort(): keys each band visited, and the weights it ran with.
     cohort_stats: dict[str, Any] | None = Field(None, serialization_alias="cohortStats")
     # Detectors the circuit breaker disabled mid-run (scan-cache identity
@@ -261,6 +265,8 @@ class RestOutputSink:
         self._scan_cache_savings: tuple[int, int] | None = None
         self._partial_coverage: bool = False
         self._partial_coverage_reason: str = ""
+        self._listing_complete: bool = False
+        self._sweep_complete: bool | None = None
         self._detectors_degraded: list[str] = []
         self._cohort_stats: dict[str, Any] | None = None
         self.relationship_report = RelationshipReport()
@@ -279,6 +285,14 @@ class RestOutputSink:
         if reason:
             self._partial_coverage_reason = str(reason)
 
+    def set_listing_complete(self) -> None:
+        """Mark that discovery named every object of the configured scope."""
+        self._listing_complete = True
+
+    def set_sweep_complete(self, complete: bool | None) -> None:
+        """Record whether the wrapping sweep has covered all its ground once."""
+        self._sweep_complete = complete
+
     def set_detectors_degraded(self, keys: list[str]) -> None:
         """Record detectors the breaker disabled mid-run, for the next run's sweep."""
         seen: set[str] = set(self._detectors_degraded)
@@ -415,6 +429,8 @@ class RestOutputSink:
             relationships_dropped=lineage.dropped,
             relationship_errors=lineage.errors or None,
             partial_coverage=True if self._partial_coverage else None,
+            discovery_complete=True if self._listing_complete else None,
+            sweep_complete=self._sweep_complete,
             cohort_stats=self._cohort_stats,
             detectors_degraded=self._detectors_degraded or None,
         )

{classifyre_cli-0.7.0 → classifyre_cli-0.7.2}/src/sources/base.py

@@ -89,6 +89,12 @@ class BaseSource(ABC):
         self._next_sampling_cursor: dict[str, Any] | None = None
         self._partial_coverage: bool = False
         self._partial_coverage_reason: str = ""
+        self._listing_complete: bool = False
+        self._listing_errors: int = 0
+        # Ground an AUTOMATIC sweep has to cover, by cursor key, and whether
+        # each unit has been covered once (see ``sweep_complete``).
+        self._sweep_units: dict[str, bool] = {}
+        self._sweep_wraps: bool = False
         self._sampling_cursor_lock = threading.Lock()
         self._apply_initial_sampling_override(normalized_recipe)
         recipe.clear()
@@ -177,6 +183,72 @@ class BaseSource(ABC):
         """
         return self._next_sampling_cursor
 
+    # ── Has the sweep gone round once? ───────────────────────────────────
+    #
+    # A cursor that wraps (an offset that returns to 0 at the end, a walk that
+    # starts over) never stops moving, by design: walking again is the only way
+    # a store that cannot be ordered by age shows its new rows. So "the cursor
+    # moved" cannot tell a sweep that is still reaching new ground from one that
+    # is going round for the second time, and a scheduler that reads it that way
+    # re-runs such a source every minute for good.
+    #
+    # Each unit of ground therefore records, once, that it has been covered: a
+    # frontier says so itself (``done``); a wrapping unit says so by wrapping
+    # (kept in the cursor under ``LAPPED_KEY``, which only ever grows). The run
+    # reports ``sweep_complete`` when every unit it touched has been covered and
+    # at least one of them wraps -- the only case the cursor cannot speak for.
+
+    #: Cursor key listing the wrapping units that have been covered once.
+    LAPPED_KEY = "_lapped"
+
+    def _lapped_keys(self) -> set[str]:
+        """Wrapping units known to have been covered once, as of this run.
+
+        Callers hold ``_sampling_cursor_lock`` when they mean to write back.
+        """
+        keys: set[str] = set()
+        for cursor in (self._sampling_cursor, self._next_sampling_cursor):
+            raw = cursor.get(self.LAPPED_KEY) if isinstance(cursor, dict) else None
+            if isinstance(raw, list):
+                keys.update(str(k) for k in raw)
+        return keys
+
+    def note_sweep_frontier(self, key: str, *, done: bool) -> None:
+        """Record whether the frontier under ``key`` has covered its ground."""
+        self._sweep_units[key] = bool(done)
+
+    def note_sweep_wrap(self, key: str, *, wrapped: bool) -> None:
+        """Record that the wrapping unit ``key`` was read this run.
+
+        ``wrapped`` says this run reached its end and starts over. That is
+        remembered in the cursor from then on, so a later run that is
+        mid-walk still counts the unit as covered.
+        """
+        with self._sampling_cursor_lock:
+            self._sweep_wraps = True
+            lapped = self._lapped_keys()
+            if wrapped and key not in lapped:
+                lapped.add(key)
+                nxt = (
+                    self._next_sampling_cursor
+                    if isinstance(self._next_sampling_cursor, dict)
+                    else dict(self._sampling_cursor)
+                )
+                self._next_sampling_cursor = {**nxt, self.LAPPED_KEY: sorted(lapped)}
+            self._sweep_units[key] = key in lapped
+
+    @property
+    def sweep_complete(self) -> bool | None:
+        """Whether every unit this run read has been covered at least once.
+
+        None when the run has nothing to say: no unit of it wraps, so the
+        cursor itself already tells a finished sweep (it stops moving).
+        """
+        units = getattr(self, "_sweep_units", None)
+        if not getattr(self, "_sweep_wraps", False) or not units:
+            return None
+        return all(units.values())
+
     def declare_partial_coverage(self, reason: str = "") -> None:
         """Say that this run looked at only part of the source.
 
@@ -191,6 +263,34 @@ class BaseSource(ABC):
         if reason and not self._partial_coverage_reason:
             self._partial_coverage_reason = str(reason)
 
+    def note_listing_error(self) -> None:
+        """Record that part of the catalog could not be listed this run.
+
+        A listing that skipped a schema, a folder or a label cannot say that
+        what it did not list is gone, so it forfeits :meth:`declare_listing_complete`.
+        """
+        self._listing_errors = getattr(self, "_listing_errors", 0) + 1
+
+    def declare_listing_complete(self) -> None:
+        """Say that discovery named every object of the configured scope.
+
+        Sampling decides how much of each asset is read, not which assets
+        exist. A catalog connector (tables, node labels) lists its whole scope
+        on every run whatever the strategy, so an asset missing from that list
+        is gone and can be retired -- without this, only strategy=ALL ever
+        retired anything and every other source kept deleted tables, moved
+        databases and renamed labels as "unchanged" forever.
+
+        Ignored when any part of the listing failed or the run declared partial
+        coverage.
+        """
+        if getattr(self, "_listing_errors", 0) == 0:
+            self._listing_complete = True
+
+    @property
+    def listing_complete(self) -> bool:
+        return bool(getattr(self, "_listing_complete", False)) and not self.partial_coverage
+
     @property
     def partial_coverage(self) -> bool:
         return bool(getattr(self, "_partial_coverage", False))
@@ -231,16 +331,24 @@ class BaseSource(ABC):
         saved = self._sampling_cursor.get(key)
         return saved if isinstance(saved, int) and not isinstance(saved, bool) and saved >= 0 else 0
 
-    def record_automatic_offset(self, key: str, *, prev_offset: int, fetched: int) -> None:
+    def record_automatic_offset(
+        self, key: str, *, prev_offset: int, fetched: int, track: bool = True
+    ) -> None:
         """Advance a keyed offset cursor; wrap to 0 once a page underfills.
 
         For a store that can only be paged by position (``skip``/``OFFSET``)
         and offers nothing to order by, so new rows cannot be told from old
         ones. Wrapping is how such a source ever sees them.
+
+        ``track=False`` keeps the unit out of ``sweep_complete``, for a caller
+        whose cursor holds state this offset knows nothing about.
         """
         size = self.sampling_window_size()
-        next_offset = 0 if fetched < size else prev_offset + fetched
+        wrapped = fetched < size
+        next_offset = 0 if wrapped else prev_offset + fetched
         self._record_cursor_key(key, next_offset)
+        if track:
+            self.note_sweep_wrap(key, wrapped=wrapped)
 
     def automatic_window(
         self,
@@ -314,7 +422,9 @@ class BaseSource(ABC):
         )
         window = items[offset : offset + size]
         next_offset = offset + len(window)
-        self._record_cursor_key(key, 0 if next_offset >= len(items) else next_offset)
+        wrapped = next_offset >= len(items)
+        self._record_cursor_key(key, 0 if wrapped else next_offset)
+        self.note_sweep_wrap(key, wrapped=wrapped)
         return [item for _, item in window]
 
     def automatic_ranged_window(
@@ -381,6 +491,7 @@ class BaseSource(ABC):
         state = frontier.to_state()
         if state is not None:
             self._record_cursor_key(key, state)
+        self.note_sweep_frontier(key, done=frontier.done or frontier.fresh)
         return window
 
     def automatic_frontier(self, key: str) -> ListFrontier:
@@ -396,6 +507,7 @@ class BaseSource(ABC):
         state = frontier.to_state()
         if state is not None:
             self._record_cursor_key(key, state)
+        self.note_sweep_frontier(key, done=frontier.done or frontier.fresh)
 
     @staticmethod
     def _read_bool_env(name: str) -> bool | None:

{classifyre_cli-0.7.0 → classifyre_cli-0.7.2}/src/sources/custom/source.py

@@ -622,7 +622,7 @@ class CustomSource(BaseSource):
         # P15). The offset is merged in, never substituted.
         self.set_next_sampling_cursor({k: v for k, v in self._sampling_cursor.items() if k != key})
         self.record_automatic_offset(
-            key, prev_offset=offset, fetched=int(frame.get("produced") or 0)
+            key, prev_offset=offset, fetched=int(frame.get("produced") or 0), track=False
         )
 
     def _handled_offset(self, frame: dict[str, Any]) -> bool:

{classifyre_cli-0.7.0 → classifyre_cli-0.7.2}/src/sources/databricks/source.py

@@ -7,7 +7,7 @@ from collections.abc import AsyncGenerator, Generator
 from contextlib import closing
 from dataclasses import dataclass
 from datetime import UTC, datetime, timedelta
-from typing import Any
+from typing import Any, ClassVar
 from urllib.parse import urlparse
 
 import requests
@@ -19,6 +19,7 @@ from ...graph.edges import (
     Method,
     Ref,
     flow,
+    same_as,
 )
 from ...models.generated_input import (
     AzureServicePrincipal,
@@ -68,6 +69,16 @@ class PipelineRef:
     state: str | None
 
 
+@dataclass(frozen=True)
+class _ForeignSystem:
+    """The database behind a foreign catalog, as its own connector names it."""
+
+    platform: str
+    authority: str
+    database: str | None
+    connection: str
+
+
 def _quote_identifier(identifier: str) -> str:
     return f"`{identifier.replace('`', '``')}`"
 
@@ -109,6 +120,10 @@ class DatabricksSource(BaseTabularSource):
         self._validate_auth_configuration()
 
         self.session = requests.Session()
+        # table_key -> upstream table -> ids of the notebooks Unity Catalog saw
+        # move data between the two. Filled by _lineage_refs_for_table.
+        self._lineage_notebook_ids: dict[tuple[str, ...], dict[tuple[str, str, str], set[str]]] = {}
+        self._foreign_systems: dict[str, _ForeignSystem | None] = {}
         self._access_token: str | None = None
         self._access_token_expiry: datetime | None = None
 
@@ -409,9 +424,13 @@ class DatabricksSource(BaseTabularSource):
         without_scheme = raw.split("://", 1)[-1]
         return without_scheme.strip("/").split("/", 1)[0]
 
-    def _available_columns(self, table_ref: TableRef) -> list[str]:
+    def _available_column_types(self, table_ref: TableRef) -> dict[str, str]:
+        # ``system.information_schema`` sees every catalog from one connection;
+        # the default query reads only the connection's current catalog and a
+        # parameter style Databricks does not take, so types used to come back
+        # empty for every column.
         query = (
-            "SELECT column_name "
+            "SELECT column_name, COALESCE(full_data_type, data_type) "
             "FROM system.information_schema.columns "
             f"WHERE table_catalog = {_quote_literal(table_ref.database)} "
             f"AND table_schema = {_quote_literal(table_ref.schema or '')} "
@@ -421,17 +440,17 @@ class DatabricksSource(BaseTabularSource):
         conn = self._get_cached_connection(table_ref.database)
         with conn.cursor() as cursor:
             cursor.execute(query)
-            columns: list[str] = []
+            columns: dict[str, str] = {}
             for row in cursor.fetchall():
-                candidate: Any | None = None
-                try:
-                    candidate = row[0]  # type: ignore[index]
-                except Exception:
-                    candidate = None
-                if isinstance(candidate, str):
-                    columns.append(candidate)
+                name = row[0] if len(row) > 0 else None
+                if isinstance(name, str):
+                    data_type = row[1] if len(row) > 1 else None
+                    columns[name] = str(data_type) if data_type is not None else ""
             return columns
 
+    def _available_columns(self, table_ref: TableRef) -> list[str]:
+        return list(self._available_column_types(table_ref))
+
     # ── Databricks-specific cell serialization ───────────────────────────
 
     def _serialize_cell(self, value: Any) -> str:
@@ -598,6 +617,7 @@ class DatabricksSource(BaseTabularSource):
         try:
             schemas = self._list_schemas_for_catalog(database)
         except Exception as exc:
+            self.note_listing_error()
             logger.warning("Skipping catalog %s due to schema listing error: %s", database, exc)
             return tables
         for schema in schemas:
@@ -606,6 +626,7 @@ class DatabricksSource(BaseTabularSource):
             try:
                 tables.extend(self._list_tables_for_schema(database, schema))
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning(
                     "Skipping schema %s.%s due to table listing error: %s",
                     database,
@@ -653,14 +674,30 @@ class DatabricksSource(BaseTabularSource):
             },
         )
         refs: set[tuple[str, str, str]] = set()
+        notebooks: dict[tuple[str, str, str], set[str]] = {}
         for key in ("upstreams", "upstream_tables"):
             entries = response.get(key)
             if isinstance(entries, list):
                 for entry in entries:
                     if isinstance(entry, dict):
+                        if (entry.get("tableInfo") or {}).get("has_permission") is False:
+                            # Unity Catalog knows the upstream and will not name
+                            # it to this identity, so the table looks sourceless.
+                            logger.warning(
+                                "%s has an upstream this identity may not see; "
+                                "grant it BROWSE on that catalog",
+                                table_ref.display_name,
+                            )
+                            continue
                         parsed = self._lineage_table_ref_from_payload(entry)
                         if parsed:
                             refs.add(parsed)
+                            for info in entry.get("notebookInfos") or []:
+                                if isinstance(info, dict) and info.get("notebook_id") is not None:
+                                    notebooks.setdefault(parsed, set()).add(
+                                        str(info["notebook_id"])
+                                    )
+        self._lineage_notebook_ids[table_ref.table_key] = notebooks
         return refs
 
     def _emit_table_lineage(
@@ -668,6 +705,7 @@ class DatabricksSource(BaseTabularSource):
         table_ref: TableRef,
         upstream_refs: set[tuple[str, str, str]],
         table_hash_by_key: dict[tuple[str, ...], str],
+        notebook_hash_by_id: dict[str, str] | None = None,
     ) -> None:
         """Turn Unity Catalog's lineage into typed edges.
 
@@ -675,6 +713,9 @@ class DatabricksSource(BaseTabularSource):
         question directly, so nothing here is parsed out of SQL — these edges
         are as trustworthy as lineage gets, and are marked SYSTEM_CATALOG.
 
+        When Unity Catalog saw a notebook run the statement, the edge carries it
+        as ``via`` -- if that notebook is one of this scan's assets.
+
         The upstreams that are *not* part of this scan are the important half.
         They used to be dropped on the floor (``if target in table_hash_by_key``),
         which meant a table fed by a catalog outside the configured scope looked
@@ -686,23 +727,52 @@ class DatabricksSource(BaseTabularSource):
             return
 
         column_mappings = self._column_mappings_for_table(table_ref)
+        notebook_ids = self._lineage_notebook_ids.get(table_ref.table_key, {})
         for upstream in sorted(upstream_refs):
             upstream_hash = table_hash_by_key.get(upstream)
-            upstream_ref = (
-                Ref.asset(upstream_hash) if upstream_hash else self._urn_ref_for_key(upstream)
-            )
+            foreign_urn = self._foreign_urn(upstream)
+            upstream_ref: Ref | None
+            if foreign_urn:
+                upstream_ref = Ref.urn(foreign_urn)
+            elif upstream_hash:
+                upstream_ref = Ref.asset(upstream_hash)
+            else:
+                upstream_ref = self._urn_ref_for_key(upstream)
             if upstream_ref is None:
                 continue
+            via_hash = next(
+                (
+                    notebook_hash_by_id[notebook_id]
+                    for notebook_id in sorted(notebook_ids.get(upstream, ()))
+                    if notebook_id in (notebook_hash_by_id or {})
+                ),
+                None,
+            )
             self.add_edge(
                 flow(
                     upstream=upstream_ref,
                     downstream=Ref.asset(downstream_hash),
                     type=FlowType.TRANSFORM,
                     fields=column_mappings.get(upstream, []),
+                    via=Ref.asset(via_hash) if via_hash else None,
                     method=Method.SYSTEM_CATALOG,
+                    evidence=self._foreign_evidence(upstream) if foreign_urn else None,
                 )
             )
 
+    def _emit_foreign_identity(self, table_ref: TableRef, asset_hash: str) -> None:
+        """A scanned table of a foreign catalog is a mirror of the real table."""
+        foreign_urn = self._foreign_urn(table_ref.table_key)
+        if not foreign_urn:
+            return
+        self.add_edge(
+            same_as(
+                Ref.asset(asset_hash),
+                Ref.urn(foreign_urn),
+                evidence=self._foreign_evidence(table_ref.table_key),
+            )
+        )
+
     def _column_mappings_for_table(
         self, table_ref: TableRef
     ) -> dict[tuple[str, str, str], list[FieldMapping]]:
@@ -744,14 +814,12 @@ class DatabricksSource(BaseTabularSource):
 
         return {
             upstream: [
+                # Unity Catalog names the columns and nothing between them, so a
+                # matching name proves nothing: `lower(email) AS email` matches.
                 FieldMapping(
                     downstream=downstream,
                     upstreams=sorted(sources),
-                    type=(
-                        FieldTransform.IDENTITY
-                        if sources == {downstream}
-                        else FieldTransform.TRANSFORMED
-                    ),
+                    type=FieldTransform.MAPPED,
                 )
                 for downstream, sources in sorted(columns.items())
             ]
@@ -767,6 +835,86 @@ class DatabricksSource(BaseTabularSource):
         except UrnError:
             return None
 
+    # ── Foreign catalogs (Lakehouse Federation) ──────────────────────────
+    #
+    # A foreign catalog is Databricks' name for tables that live in another
+    # database. Unity Catalog records lineage through it like any other catalog,
+    # so without this the upstream is `databricks://.../app_db/core/customers`,
+    # a name no PostgreSQL scan will ever write, and the path stops at the
+    # boundary it was most worth crossing.
+
+    #: UC connection type -> (URN platform, catalog option naming the database).
+    #: None where the schema level already is the database, as in MySQL.
+    _FOREIGN_PLATFORMS: ClassVar[dict[str, tuple[str, str | None]]] = {
+        "POSTGRESQL": ("postgres", "database"),
+        "MYSQL": ("mysql", None),
+        "SQLSERVER": ("mssql", "database"),
+        "ORACLE": ("oracle", "service_name"),
+        "SNOWFLAKE": ("snowflake", "database"),
+    }
+
+    def _foreign_system(self, catalog: str) -> _ForeignSystem | None:
+        if catalog in self._foreign_systems:
+            return self._foreign_systems[catalog]
+        system: _ForeignSystem | None = None
+        try:
+            system = self._read_foreign_system(catalog)
+        except Exception as exc:
+            # USE CONNECTION is a separate grant. The edge keeps its Databricks
+            # name, which is true if not useful, rather than the scan failing.
+            logger.warning("Could not read the connection of catalog %s: %s", catalog, exc)
+        self._foreign_systems[catalog] = system
+        return system
+
+    def _read_foreign_system(self, catalog: str) -> _ForeignSystem | None:
+        info = self._request_json("get", f"/api/2.1/unity-catalog/catalogs/{catalog}")
+        connection_name = info.get("connection_name")
+        if info.get("catalog_type") != "FOREIGN_CATALOG" or not connection_name:
+            return None
+        connection = self._request_json(
+            "get", f"/api/2.1/unity-catalog/connections/{connection_name}"
+        )
+        mapping = self._FOREIGN_PLATFORMS.get(str(connection.get("connection_type")))
+        if mapping is None:
+            return None
+        platform, database_option = mapping
+
+        options = connection.get("options") or {}
+        host = str(options.get("host") or "").strip()
+        if not host:
+            # A URN scoped to the wrong host would stitch two unrelated systems.
+            return None
+        if platform == "snowflake":
+            authority = host.removesuffix(".snowflakecomputing.com")
+        else:
+            port = options.get("port")
+            authority = f"{host}:{port}" if port else host
+
+        database = None
+        if database_option:
+            database = str((info.get("options") or {}).get(database_option) or "").strip()
+            if not database:
+                return None
+        return _ForeignSystem(platform, authority, database, str(connection_name))
+
+    def _foreign_urn(self, table_key: tuple[str, ...]) -> str | None:
+        """The URN the underlying system's own connector writes for this table."""
+        system = self._foreign_system(table_key[0])
+        if system is None:
+            return None
+        name = (system.database, *table_key[1:]) if system.database else table_key[1:]
+        try:
+            return str(Urn.of(system.platform, system.authority, *name))
+        except UrnError:
+            return None
+
+    def _foreign_evidence(self, table_key: tuple[str, ...]) -> dict[str, Any]:
+        system = self._foreign_system(table_key[0])
+        return {
+            "foreign_catalog": table_key[0],
+            "connection": system.connection if system else None,
+        }
+
     # ── Notebooks ────────────────────────────────────────────────────────
 
     def _iter_notebooks(self) -> Generator[NotebookRef, None, None]:
@@ -786,6 +934,7 @@ class DatabricksSource(BaseTabularSource):
                     "get", "/api/2.0/workspace/list", params={"path": path}
                 )
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning("Skipping workspace path %s due to listing error: %s", path, exc)
                 continue
             objects = payload.get("objects")
@@ -865,6 +1014,7 @@ class DatabricksSource(BaseTabularSource):
             try:
                 payload = self._request_json("get", "/api/2.0/pipelines", params=params)
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning("Could not list Databricks pipelines: %s", exc)
                 break
             values: list[dict[str, Any]] = []
@@ -935,6 +1085,15 @@ class DatabricksSource(BaseTabularSource):
             t.table_key: self.generate_hash_id(t.raw_id) for t in tables
         }
 
+        # Listed before the tables so a lineage edge can name the notebook that
+        # produced it; the notebook assets are still emitted after the tables.
+        notebooks = list(self._iter_notebooks())
+        notebook_hash_by_id = {
+            nb.object_id: self.generate_hash_id(f"notebook_#_{nb.path}")
+            for nb in notebooks
+            if nb.object_id
+        }
+
         include_lineage = self._extraction_options().include_table_lineage
         if include_lineage and tables:
             logger.info("Fetching table lineage for %d table(s)...", len(tables))
@@ -957,7 +1116,9 @@ class DatabricksSource(BaseTabularSource):
                         for target in sorted(upstream_refs)
                         if target in table_hash_by_key
                     ]
-                    self._emit_table_lineage(table_ref, upstream_refs, table_hash_by_key)
+                    self._emit_table_lineage(
+                        table_ref, upstream_refs, table_hash_by_key, notebook_hash_by_id
+                    )
                 except Exception as exc:
                     logger.warning(
                         "Could not resolve Databricks lineage for %s: %s", table_label, exc
@@ -967,6 +1128,8 @@ class DatabricksSource(BaseTabularSource):
             self._table_lookup[asset.hash] = table_ref
             batch.append(asset)
             emitted_tables += 1
+            if include_lineage:
+                self._emit_foreign_identity(table_ref, asset.hash)
 
             if len(batch) >= self.BATCH_SIZE:
                 yield batch
@@ -974,7 +1137,7 @@ class DatabricksSource(BaseTabularSource):
 
         # Notebooks
         notebook_count = 0
-        for notebook in self._iter_notebooks():
+        for notebook in notebooks:
             if self._aborted:
                 break
             batch.append(self._notebook_to_asset(notebook))
@@ -997,6 +1160,7 @@ class DatabricksSource(BaseTabularSource):
                 batch = []
         if pipeline_count:
             logger.info("Discovered %d pipeline(s)", pipeline_count)
+        self._declare_listing_if_whole()
 
         if batch:
             yield batch

{classifyre_cli-0.7.0 → classifyre_cli-0.7.2}/src/sources/hive/source.py

@@ -293,7 +293,10 @@ class HiveSource(BaseTabularSource):
             upstreams = [
                 key
                 for key in upstream_tables_from_sql(
-                    definition, dialect="hive", default_database=view.database
+                    definition,
+                    dialect="hive",
+                    default_database=view.database,
+                    two_level=True,
                 )
                 if key != view.table_key and key in scoped
             ]