classifyre-cli 0.7.1__tar.gz → 0.7.2__tar.gz

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 325 packages in 329ms
+Resolved 325 packages in 204ms
 Checked 52 packages in 4ms

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.7.1
+Version: 0.7.2
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.7.1",
+  "version": "0.7.2",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.7.1"
+version = "0.7.2"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/src/graph/edges.py

@@ -131,6 +131,7 @@ class FieldTransform(StrEnum):
     IDENTITY = "IDENTITY"  # copied through unchanged
     TRANSFORMED = "TRANSFORMED"  # an expression combined the upstreams
     AGGREGATED = "AGGREGATED"  # SUM/COUNT/... over a group
+    MAPPED = "MAPPED"  # the platform says it feeds this column, not how
     INDIRECT = "INDIRECT"  # influenced the rows, not the values
 
 

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/src/main.py

@@ -976,6 +976,9 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                     ):
                         sink.set_listing_complete()
 
+                    if hasattr(sink, "set_sweep_complete"):
+                        sink.set_sweep_complete(getattr(source, "sweep_complete", None))
+
                     # Relationships go *before* finish(): finish() finalizes the
                     # run and marks it COMPLETED, and edges that land after that
                     # belong to a run that already claimed to be done.

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/src/outputs/rest.py

@@ -204,6 +204,9 @@ class FinalizeIngestRunRequest(BaseModel):
     # from a cohort run is not evidence that an asset is gone.
     partial_coverage: bool | None = Field(None, serialization_alias="partialCoverage")
     discovery_complete: bool | None = Field(None, serialization_alias="discoveryComplete")
+    # Whether every unit of an AUTOMATIC sweep that wraps has been covered once.
+    # Only sent by a run with a wrapping unit; see BaseSource.sweep_complete.
+    sweep_complete: bool | None = Field(None, serialization_alias="sweepComplete")
     # Per ctx.cohort(): keys each band visited, and the weights it ran with.
     cohort_stats: dict[str, Any] | None = Field(None, serialization_alias="cohortStats")
     # Detectors the circuit breaker disabled mid-run (scan-cache identity
@@ -263,6 +266,7 @@ class RestOutputSink:
         self._partial_coverage: bool = False
         self._partial_coverage_reason: str = ""
         self._listing_complete: bool = False
+        self._sweep_complete: bool | None = None
         self._detectors_degraded: list[str] = []
         self._cohort_stats: dict[str, Any] | None = None
         self.relationship_report = RelationshipReport()
@@ -285,6 +289,10 @@ class RestOutputSink:
         """Mark that discovery named every object of the configured scope."""
         self._listing_complete = True
 
+    def set_sweep_complete(self, complete: bool | None) -> None:
+        """Record whether the wrapping sweep has covered all its ground once."""
+        self._sweep_complete = complete
+
     def set_detectors_degraded(self, keys: list[str]) -> None:
         """Record detectors the breaker disabled mid-run, for the next run's sweep."""
         seen: set[str] = set(self._detectors_degraded)
@@ -422,6 +430,7 @@ class RestOutputSink:
             relationship_errors=lineage.errors or None,
             partial_coverage=True if self._partial_coverage else None,
             discovery_complete=True if self._listing_complete else None,
+            sweep_complete=self._sweep_complete,
             cohort_stats=self._cohort_stats,
             detectors_degraded=self._detectors_degraded or None,
         )

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/src/sources/base.py

@@ -91,6 +91,10 @@ class BaseSource(ABC):
         self._partial_coverage_reason: str = ""
         self._listing_complete: bool = False
         self._listing_errors: int = 0
+        # Ground an AUTOMATIC sweep has to cover, by cursor key, and whether
+        # each unit has been covered once (see ``sweep_complete``).
+        self._sweep_units: dict[str, bool] = {}
+        self._sweep_wraps: bool = False
         self._sampling_cursor_lock = threading.Lock()
         self._apply_initial_sampling_override(normalized_recipe)
         recipe.clear()
@@ -179,6 +183,72 @@ class BaseSource(ABC):
         """
         return self._next_sampling_cursor
 
+    # ── Has the sweep gone round once? ───────────────────────────────────
+    #
+    # A cursor that wraps (an offset that returns to 0 at the end, a walk that
+    # starts over) never stops moving, by design: walking again is the only way
+    # a store that cannot be ordered by age shows its new rows. So "the cursor
+    # moved" cannot tell a sweep that is still reaching new ground from one that
+    # is going round for the second time, and a scheduler that reads it that way
+    # re-runs such a source every minute for good.
+    #
+    # Each unit of ground therefore records, once, that it has been covered: a
+    # frontier says so itself (``done``); a wrapping unit says so by wrapping
+    # (kept in the cursor under ``LAPPED_KEY``, which only ever grows). The run
+    # reports ``sweep_complete`` when every unit it touched has been covered and
+    # at least one of them wraps -- the only case the cursor cannot speak for.
+
+    #: Cursor key listing the wrapping units that have been covered once.
+    LAPPED_KEY = "_lapped"
+
+    def _lapped_keys(self) -> set[str]:
+        """Wrapping units known to have been covered once, as of this run.
+
+        Callers hold ``_sampling_cursor_lock`` when they mean to write back.
+        """
+        keys: set[str] = set()
+        for cursor in (self._sampling_cursor, self._next_sampling_cursor):
+            raw = cursor.get(self.LAPPED_KEY) if isinstance(cursor, dict) else None
+            if isinstance(raw, list):
+                keys.update(str(k) for k in raw)
+        return keys
+
+    def note_sweep_frontier(self, key: str, *, done: bool) -> None:
+        """Record whether the frontier under ``key`` has covered its ground."""
+        self._sweep_units[key] = bool(done)
+
+    def note_sweep_wrap(self, key: str, *, wrapped: bool) -> None:
+        """Record that the wrapping unit ``key`` was read this run.
+
+        ``wrapped`` says this run reached its end and starts over. That is
+        remembered in the cursor from then on, so a later run that is
+        mid-walk still counts the unit as covered.
+        """
+        with self._sampling_cursor_lock:
+            self._sweep_wraps = True
+            lapped = self._lapped_keys()
+            if wrapped and key not in lapped:
+                lapped.add(key)
+                nxt = (
+                    self._next_sampling_cursor
+                    if isinstance(self._next_sampling_cursor, dict)
+                    else dict(self._sampling_cursor)
+                )
+                self._next_sampling_cursor = {**nxt, self.LAPPED_KEY: sorted(lapped)}
+            self._sweep_units[key] = key in lapped
+
+    @property
+    def sweep_complete(self) -> bool | None:
+        """Whether every unit this run read has been covered at least once.
+
+        None when the run has nothing to say: no unit of it wraps, so the
+        cursor itself already tells a finished sweep (it stops moving).
+        """
+        units = getattr(self, "_sweep_units", None)
+        if not getattr(self, "_sweep_wraps", False) or not units:
+            return None
+        return all(units.values())
+
     def declare_partial_coverage(self, reason: str = "") -> None:
         """Say that this run looked at only part of the source.
 
@@ -261,16 +331,24 @@ class BaseSource(ABC):
         saved = self._sampling_cursor.get(key)
         return saved if isinstance(saved, int) and not isinstance(saved, bool) and saved >= 0 else 0
 
-    def record_automatic_offset(self, key: str, *, prev_offset: int, fetched: int) -> None:
+    def record_automatic_offset(
+        self, key: str, *, prev_offset: int, fetched: int, track: bool = True
+    ) -> None:
         """Advance a keyed offset cursor; wrap to 0 once a page underfills.
 
         For a store that can only be paged by position (``skip``/``OFFSET``)
         and offers nothing to order by, so new rows cannot be told from old
         ones. Wrapping is how such a source ever sees them.
+
+        ``track=False`` keeps the unit out of ``sweep_complete``, for a caller
+        whose cursor holds state this offset knows nothing about.
         """
         size = self.sampling_window_size()
-        next_offset = 0 if fetched < size else prev_offset + fetched
+        wrapped = fetched < size
+        next_offset = 0 if wrapped else prev_offset + fetched
         self._record_cursor_key(key, next_offset)
+        if track:
+            self.note_sweep_wrap(key, wrapped=wrapped)
 
     def automatic_window(
         self,
@@ -344,7 +422,9 @@ class BaseSource(ABC):
         )
         window = items[offset : offset + size]
         next_offset = offset + len(window)
-        self._record_cursor_key(key, 0 if next_offset >= len(items) else next_offset)
+        wrapped = next_offset >= len(items)
+        self._record_cursor_key(key, 0 if wrapped else next_offset)
+        self.note_sweep_wrap(key, wrapped=wrapped)
         return [item for _, item in window]
 
     def automatic_ranged_window(
@@ -411,6 +491,7 @@ class BaseSource(ABC):
         state = frontier.to_state()
         if state is not None:
             self._record_cursor_key(key, state)
+        self.note_sweep_frontier(key, done=frontier.done or frontier.fresh)
         return window
 
     def automatic_frontier(self, key: str) -> ListFrontier:
@@ -426,6 +507,7 @@ class BaseSource(ABC):
         state = frontier.to_state()
         if state is not None:
             self._record_cursor_key(key, state)
+        self.note_sweep_frontier(key, done=frontier.done or frontier.fresh)
 
     @staticmethod
     def _read_bool_env(name: str) -> bool | None:

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/src/sources/custom/source.py

@@ -622,7 +622,7 @@ class CustomSource(BaseSource):
         # P15). The offset is merged in, never substituted.
         self.set_next_sampling_cursor({k: v for k, v in self._sampling_cursor.items() if k != key})
         self.record_automatic_offset(
-            key, prev_offset=offset, fetched=int(frame.get("produced") or 0)
+            key, prev_offset=offset, fetched=int(frame.get("produced") or 0), track=False
         )
 
     def _handled_offset(self, frame: dict[str, Any]) -> bool:

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/src/sources/databricks/source.py

@@ -7,7 +7,7 @@ from collections.abc import AsyncGenerator, Generator
 from contextlib import closing
 from dataclasses import dataclass
 from datetime import UTC, datetime, timedelta
-from typing import Any
+from typing import Any, ClassVar
 from urllib.parse import urlparse
 
 import requests
@@ -19,6 +19,7 @@ from ...graph.edges import (
     Method,
     Ref,
     flow,
+    same_as,
 )
 from ...models.generated_input import (
     AzureServicePrincipal,
@@ -68,6 +69,16 @@ class PipelineRef:
     state: str | None
 
 
+@dataclass(frozen=True)
+class _ForeignSystem:
+    """The database behind a foreign catalog, as its own connector names it."""
+
+    platform: str
+    authority: str
+    database: str | None
+    connection: str
+
+
 def _quote_identifier(identifier: str) -> str:
     return f"`{identifier.replace('`', '``')}`"
 
@@ -112,6 +123,7 @@ class DatabricksSource(BaseTabularSource):
         # table_key -> upstream table -> ids of the notebooks Unity Catalog saw
         # move data between the two. Filled by _lineage_refs_for_table.
         self._lineage_notebook_ids: dict[tuple[str, ...], dict[tuple[str, str, str], set[str]]] = {}
+        self._foreign_systems: dict[str, _ForeignSystem | None] = {}
         self._access_token: str | None = None
         self._access_token_expiry: datetime | None = None
 
@@ -668,6 +680,15 @@ class DatabricksSource(BaseTabularSource):
             if isinstance(entries, list):
                 for entry in entries:
                     if isinstance(entry, dict):
+                        if (entry.get("tableInfo") or {}).get("has_permission") is False:
+                            # Unity Catalog knows the upstream and will not name
+                            # it to this identity, so the table looks sourceless.
+                            logger.warning(
+                                "%s has an upstream this identity may not see; "
+                                "grant it BROWSE on that catalog",
+                                table_ref.display_name,
+                            )
+                            continue
                         parsed = self._lineage_table_ref_from_payload(entry)
                         if parsed:
                             refs.add(parsed)
@@ -709,9 +730,14 @@ class DatabricksSource(BaseTabularSource):
         notebook_ids = self._lineage_notebook_ids.get(table_ref.table_key, {})
         for upstream in sorted(upstream_refs):
             upstream_hash = table_hash_by_key.get(upstream)
-            upstream_ref = (
-                Ref.asset(upstream_hash) if upstream_hash else self._urn_ref_for_key(upstream)
-            )
+            foreign_urn = self._foreign_urn(upstream)
+            upstream_ref: Ref | None
+            if foreign_urn:
+                upstream_ref = Ref.urn(foreign_urn)
+            elif upstream_hash:
+                upstream_ref = Ref.asset(upstream_hash)
+            else:
+                upstream_ref = self._urn_ref_for_key(upstream)
             if upstream_ref is None:
                 continue
             via_hash = next(
@@ -730,9 +756,23 @@ class DatabricksSource(BaseTabularSource):
                     fields=column_mappings.get(upstream, []),
                     via=Ref.asset(via_hash) if via_hash else None,
                     method=Method.SYSTEM_CATALOG,
+                    evidence=self._foreign_evidence(upstream) if foreign_urn else None,
                 )
             )
 
+    def _emit_foreign_identity(self, table_ref: TableRef, asset_hash: str) -> None:
+        """A scanned table of a foreign catalog is a mirror of the real table."""
+        foreign_urn = self._foreign_urn(table_ref.table_key)
+        if not foreign_urn:
+            return
+        self.add_edge(
+            same_as(
+                Ref.asset(asset_hash),
+                Ref.urn(foreign_urn),
+                evidence=self._foreign_evidence(table_ref.table_key),
+            )
+        )
+
     def _column_mappings_for_table(
         self, table_ref: TableRef
     ) -> dict[tuple[str, str, str], list[FieldMapping]]:
@@ -774,14 +814,12 @@ class DatabricksSource(BaseTabularSource):
 
         return {
             upstream: [
+                # Unity Catalog names the columns and nothing between them, so a
+                # matching name proves nothing: `lower(email) AS email` matches.
                 FieldMapping(
                     downstream=downstream,
                     upstreams=sorted(sources),
-                    type=(
-                        FieldTransform.IDENTITY
-                        if sources == {downstream}
-                        else FieldTransform.TRANSFORMED
-                    ),
+                    type=FieldTransform.MAPPED,
                 )
                 for downstream, sources in sorted(columns.items())
             ]
@@ -797,6 +835,86 @@ class DatabricksSource(BaseTabularSource):
         except UrnError:
             return None
 
+    # ── Foreign catalogs (Lakehouse Federation) ──────────────────────────
+    #
+    # A foreign catalog is Databricks' name for tables that live in another
+    # database. Unity Catalog records lineage through it like any other catalog,
+    # so without this the upstream is `databricks://.../app_db/core/customers`,
+    # a name no PostgreSQL scan will ever write, and the path stops at the
+    # boundary it was most worth crossing.
+
+    #: UC connection type -> (URN platform, catalog option naming the database).
+    #: None where the schema level already is the database, as in MySQL.
+    _FOREIGN_PLATFORMS: ClassVar[dict[str, tuple[str, str | None]]] = {
+        "POSTGRESQL": ("postgres", "database"),
+        "MYSQL": ("mysql", None),
+        "SQLSERVER": ("mssql", "database"),
+        "ORACLE": ("oracle", "service_name"),
+        "SNOWFLAKE": ("snowflake", "database"),
+    }
+
+    def _foreign_system(self, catalog: str) -> _ForeignSystem | None:
+        if catalog in self._foreign_systems:
+            return self._foreign_systems[catalog]
+        system: _ForeignSystem | None = None
+        try:
+            system = self._read_foreign_system(catalog)
+        except Exception as exc:
+            # USE CONNECTION is a separate grant. The edge keeps its Databricks
+            # name, which is true if not useful, rather than the scan failing.
+            logger.warning("Could not read the connection of catalog %s: %s", catalog, exc)
+        self._foreign_systems[catalog] = system
+        return system
+
+    def _read_foreign_system(self, catalog: str) -> _ForeignSystem | None:
+        info = self._request_json("get", f"/api/2.1/unity-catalog/catalogs/{catalog}")
+        connection_name = info.get("connection_name")
+        if info.get("catalog_type") != "FOREIGN_CATALOG" or not connection_name:
+            return None
+        connection = self._request_json(
+            "get", f"/api/2.1/unity-catalog/connections/{connection_name}"
+        )
+        mapping = self._FOREIGN_PLATFORMS.get(str(connection.get("connection_type")))
+        if mapping is None:
+            return None
+        platform, database_option = mapping
+
+        options = connection.get("options") or {}
+        host = str(options.get("host") or "").strip()
+        if not host:
+            # A URN scoped to the wrong host would stitch two unrelated systems.
+            return None
+        if platform == "snowflake":
+            authority = host.removesuffix(".snowflakecomputing.com")
+        else:
+            port = options.get("port")
+            authority = f"{host}:{port}" if port else host
+
+        database = None
+        if database_option:
+            database = str((info.get("options") or {}).get(database_option) or "").strip()
+            if not database:
+                return None
+        return _ForeignSystem(platform, authority, database, str(connection_name))
+
+    def _foreign_urn(self, table_key: tuple[str, ...]) -> str | None:
+        """The URN the underlying system's own connector writes for this table."""
+        system = self._foreign_system(table_key[0])
+        if system is None:
+            return None
+        name = (system.database, *table_key[1:]) if system.database else table_key[1:]
+        try:
+            return str(Urn.of(system.platform, system.authority, *name))
+        except UrnError:
+            return None
+
+    def _foreign_evidence(self, table_key: tuple[str, ...]) -> dict[str, Any]:
+        system = self._foreign_system(table_key[0])
+        return {
+            "foreign_catalog": table_key[0],
+            "connection": system.connection if system else None,
+        }
+
     # ── Notebooks ────────────────────────────────────────────────────────
 
     def _iter_notebooks(self) -> Generator[NotebookRef, None, None]:
@@ -1010,6 +1128,8 @@ class DatabricksSource(BaseTabularSource):
             self._table_lookup[asset.hash] = table_ref
             batch.append(asset)
             emitted_tables += 1
+            if include_lineage:
+                self._emit_foreign_identity(table_ref, asset.hash)
 
             if len(batch) >= self.BATCH_SIZE:
                 yield batch

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/src/sources/mongodb/source.py

@@ -507,6 +507,7 @@ class MongoDBSource(BaseSource):
                 if next_state.get(name) is not None:
                     next_state[name] = self._encode_id(next_state[name])
             self._record_cursor_key(key, next_state)
+        self.note_sweep_frontier(key, done=frontier.done or frontier.fresh)
         return documents
 
     def _sample_collection_documents(self, collection_ref: CollectionRef) -> list[dict[str, Any]]:

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/src/sources/tabular_base.py

@@ -703,6 +703,18 @@ class BaseTabularSource(BaseSource):
             nxt["tables"] = tables
             self._next_sampling_cursor = nxt
             self._automatic_advanced.add(key)
+            self._note_table_sweep(key, state)
+
+    def _note_table_sweep(self, key: str, state: dict[str, Any] | None) -> None:
+        """Tell the sweep tracker whether this table has been covered once."""
+        if state is None:
+            self.note_sweep_frontier(key, done=True)
+        elif state.get("mode") == self._WRAP_MODE:
+            # Walked and wrapped: covered once the walk has reached the end.
+            self._sweep_wraps = True
+            self._sweep_units[key] = bool(state.get("lapped"))
+        else:
+            self.note_sweep_frontier(key, done=bool(state.get("done")) or not state.get("head"))
 
     def _fetch_page_key_range(
         self,
@@ -850,6 +862,7 @@ class BaseTabularSource(BaseSource):
             )
             # Exhausted (or a key that cannot be stored, e.g. binary) leaves no
             # position, so the next run starts the walk again.
+            exhausted = len(rows) < rows_per_page
             if rows and len(rows) >= rows_per_page:
                 last_row = rows[-1]
                 position = [
@@ -860,8 +873,13 @@ class BaseTabularSource(BaseSource):
         else:
             offset = int(saved.get("offset") or 0)
             rows, column_names = self._fetch_one_page(table_ref, base_query, rows_per_page, offset)
+            exhausted = len(rows) < rows_per_page
             if rows and len(rows) >= rows_per_page:
                 next_state["offset"] = offset + len(rows)
+        # Sticky: once the walk has reached the end, the table counts as
+        # covered however many times it is walked again.
+        if exhausted or saved.get("lapped"):
+            next_state["lapped"] = True
 
         self._record_table_cursor(key, next_state)
 

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/tests/test_automatic_new_first.py

@@ -378,7 +378,7 @@ def test_mongodb_wraps_when_ids_do_not_order_by_age(monkeypatch: pytest.MonkeyPa
     assert cursor == {"collection:app.events": 10}
     second, cursor = _mongo_run(monkeypatch, collection, cursor)
     assert len(first) == 10 and len(second) == 5
-    assert cursor == {"collection:app.events": 0}
+    assert cursor == {"collection:app.events": 0, "_lapped": ["collection:app.events"]}
     # Round again: with nothing to tell new from old, this is how new ones are seen.
     assert len(_mongo_run(monkeypatch, collection, cursor)[0]) == 10
 

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/tests/test_databricks_source.py

@@ -487,6 +487,150 @@ def test_databricks_lineage_edge_names_the_notebook_that_wrote_it(
     assert edges["h-payments"].via_id is None
 
 
+def _foreign_catalogs(path: str) -> dict[str, Any]:
+    return {
+        "/api/2.1/unity-catalog/catalogs/app_db": {
+            "catalog_type": "FOREIGN_CATALOG",
+            "connection_name": "app_conn",
+            "options": {"database": "shop"},
+        },
+        "/api/2.1/unity-catalog/connections/app_conn": {
+            "connection_type": "POSTGRESQL",
+            "options": {"host": "pg.example.test", "port": "5432"},
+        },
+        "/api/2.1/unity-catalog/catalogs/hr_db": {
+            "catalog_type": "FOREIGN_CATALOG",
+            "connection_name": "hr_conn",
+        },
+        "/api/2.1/unity-catalog/connections/hr_conn": {
+            "connection_type": "MYSQL",
+            "options": {"host": "my.example.test", "port": "10218"},
+        },
+        "/api/2.1/unity-catalog/catalogs/main": {"catalog_type": "MANAGED_CATALOG"},
+    }[path]
+
+
+def test_databricks_foreign_catalog_upstream_is_named_by_its_real_system(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    source = DatabricksSource(_pat_recipe())
+    monkeypatch.setattr(source, "_request_json", lambda _m, path, **_k: _foreign_catalogs(path))
+    downstream = TableRef(database="main", schema="mart", table="daily", object_type="TABLE")
+
+    source._emit_table_lineage(
+        downstream,
+        {("app_db", "core", "customers"), ("hr_db", "people", "Staff")},
+        {downstream.table_key: "h-daily"},
+    )
+
+    urns = {edge.from_urn: edge for edge in source.drain_edges()}
+    # What the PostgreSQL and MySQL connectors write for the same tables.
+    assert set(urns) == {
+        "postgres://pg.example.test/shop/core/customers",
+        "mysql://my.example.test:10218/people/staff",
+    }
+    evidence = urns["postgres://pg.example.test/shop/core/customers"].evidence
+    assert evidence["foreign_catalog"] == "app_db"
+    assert evidence["connection"] == "app_conn"
+
+
+def test_databricks_scanned_foreign_table_is_tied_to_the_table_it_mirrors(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    source = DatabricksSource(_pat_recipe())
+    monkeypatch.setattr(source, "_request_json", lambda _m, path, **_k: _foreign_catalogs(path))
+    downstream = TableRef(database="main", schema="mart", table="daily", object_type="TABLE")
+    foreign = TableRef(database="app_db", schema="core", table="orders", object_type="TABLE")
+
+    source._emit_table_lineage(
+        downstream,
+        {foreign.table_key},
+        {downstream.table_key: "h-daily", foreign.table_key: "h-foreign"},
+    )
+    source._emit_foreign_identity(foreign, "h-foreign")
+
+    edges = source.drain_edges()
+    # Lineage goes straight to the real table, and the mirror is the same thing.
+    assert [(e.relation_class, e.from_urn) for e in edges if e.relation_class == "FLOW"] == [
+        ("FLOW", "postgres://pg.example.test/shop/core/orders")
+    ]
+    same = [e for e in edges if e.relation_class == "IDENTITY"]
+    assert [(e.from_hash, e.to_urn) for e in same] == [
+        ("h-foreign", "postgres://pg.example.test/shop/core/orders")
+    ]
+
+
+def test_databricks_foreign_catalog_it_cannot_read_stays_a_databricks_name(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    # No USE CONNECTION for the scan principal: the edge must survive unbound.
+    source = DatabricksSource(_pat_recipe())
+
+    def _denied(_method: str, path: str, **_k: Any) -> dict[str, Any]:
+        if "/connections/" in path:
+            raise RuntimeError("PERMISSION_DENIED")
+        return _foreign_catalogs(path)
+
+    monkeypatch.setattr(source, "_request_json", _denied)
+    downstream = TableRef(database="main", schema="mart", table="daily", object_type="TABLE")
+
+    source._emit_table_lineage(
+        downstream, {("app_db", "core", "customers")}, {downstream.table_key: "h-daily"}
+    )
+
+    assert [edge.from_urn for edge in source.drain_edges()] == [
+        "databricks://adb-123410678901234106.7.azuredatabricks.net/app_db/core/customers"
+    ]
+
+
+def test_databricks_column_lineage_does_not_claim_a_copy_from_a_matching_name(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    source = DatabricksSource(
+        _pat_recipe(
+            optional={"extraction": {"include_table_lineage": True, "include_column_lineage": True}}
+        )
+    )
+    downstream = TableRef(database="main", schema="mart", table="daily", object_type="TABLE")
+    monkeypatch.setattr(source, "_cached_columns", lambda _t: ["email"])
+    monkeypatch.setattr(
+        source,
+        "_request_json",
+        lambda *_a, **_k: {
+            "upstream_cols": [
+                {
+                    "catalog_name": "main",
+                    "schema_name": "raw",
+                    "table_name": "users",
+                    "name": "email",
+                }
+            ]
+        },
+    )
+
+    mappings = source._column_mappings_for_table(downstream)
+
+    assert [m.type.value for m in mappings[("main", "raw", "users")]] == ["MAPPED"]
+
+
+def test_databricks_upstream_hidden_by_permissions_is_reported(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    source = DatabricksSource(_pat_recipe())
+    downstream = TableRef(database="main", schema="mart", table="daily", object_type="TABLE")
+    monkeypatch.setattr(
+        source,
+        "_request_json",
+        lambda *_a, **_k: {"upstreams": [{"tableInfo": {"has_permission": False}}]},
+    )
+
+    with caplog.at_level("WARNING"):
+        refs = source._lineage_refs_for_table(downstream)
+
+    assert refs == set()
+    assert "grant it BROWSE" in caplog.text
+
+
 def test_databricks_unlistable_schema_forfeits_the_complete_declaration(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

{classifyre_cli-0.7.1 → classifyre_cli-0.7.2}/tests/test_elasticsearch_source.py

@@ -245,7 +245,7 @@ def test_elasticsearch_automatic_strategy_resumes_and_wraps_on_underfill(
     assert body["from"] == 10
     # Only 2 of 12 docs remain from offset 10 → underfilled page → wraps to 0.
     assert len(docs) == 2
-    assert src.current_sampling_cursor() == {"index:orders": 0}
+    assert src.current_sampling_cursor() == {"index:orders": 0, "_lapped": ["index:orders"]}
 
 
 async def test_elasticsearch_all_strategy_scrolls_the_whole_index(