classifyre-cli 0.7.0__py3-none-any.whl → 0.7.2__py3-none-any.whl

{classifyre_cli-0.7.0.dist-info → classifyre_cli-0.7.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.7.0
+Version: 0.7.2
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.7.0.dist-info → classifyre_cli-0.7.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 src/__init__.py,sha256=a9znVQMuKTEdH6nB8kUs7jMmPV1U6IFXDzCU5RnzxFo,23
 src/config.py,sha256=c4v6RbUQuWNIaCySaL5isNmBcUnYZimaHBW7Ki2InpU,2688
-src/main.py,sha256=NQzhDGS-_ycYdN_gfg_qK7es-I-h3DnBkFwwTWWYkXE,61287
+src/main.py,sha256=lcc7d9rG0TQa5TcLadRqhH77bx5ulpqLyJ2dqQqHs-Y,61645
 src/telemetry.py,sha256=fbSmuJ_t-kSthfIh3E7ESVkloH22kzl8lQKY370YtuU,3089
 src/augmentation/__init__.py,sha256=pfb4zDjcyo62kHtvRl9QJ-xV28K0A_DTT_kJoF559Lc,147
 src/augmentation/contract.py,sha256=V0XQn7gk--vRSFTt7f5nC2fh8568KQR_oZ2hg3Tf20I,2716
@@ -53,7 +53,7 @@ src/file_evaluation/__init__.py,sha256=S2IF_CeZtxbfcVKcEamdyW1mZVb-fHAuWqt8QE999
 src/file_evaluation/asset_fixture.py,sha256=6fcEXlIxYYVE7xJpGtH28Zhmwjdh4ewxqfJlCJSOZ8Y,5192
 src/file_evaluation/runner.py,sha256=_hZ9ewrnnU-QkUmK19MErA57AZ06TQqAzncoeVc9eOM,17379
 src/graph/__init__.py,sha256=IQYUoqc3x5ZF49EBD_uOqyGF42R3JpQkmSZ9oN09ezA,609
-src/graph/edges.py,sha256=OIc795_PnSvRl_fRfGevsf0tBKP2zJWqUTcOc_Bakx4,18123
+src/graph/edges.py,sha256=svktVn6M_aiiYnAPO2lLJJiyegoA65EAsgDX3vsHccM,18196
 src/models/generated_detectors.py,sha256=I-hHQ1iPM7NzWRE5DDamcWB0noNSUq44Lnubco_V3UA,73164
 src/models/generated_input.py,sha256=3_BI2n4NJ4Md9OcTZTeVYxT-bRSb8Q5pK6CFbNEuv_k,214174
 src/models/generated_single_asset_scan_results.py,sha256=zJ78x8GsjpiOBWG5aOQHHVBHdjQeosmCPQ2etwYxAv4,16508
@@ -77,7 +77,7 @@ src/outputs/base.py,sha256=uMc0_o2FhA9Y8wlwpg75ljHaXIkF-Yh_frZcPNZVcMo,4962
 src/outputs/console.py,sha256=OwPP7fXCWFML2zB4d8AxPXikanebI8CrixbgsS3CMGM,1803
 src/outputs/factory.py,sha256=ritKokn9SM6s51MSCPJhLKQMwhKF4OBRatsntGYi22w,5397
 src/outputs/file.py,sha256=6y585mxJS04MgTcMu5KPkJt_hm9qLjnbPEzD4MgrmI4,2594
-src/outputs/rest.py,sha256=-kzbkrsHrijFF9OaEvUvA29MBoQFyAp9q0kyT8k0W7g,28107
+src/outputs/rest.py,sha256=o2e9SXxNIjX6y5dSGmyY-QjQbI1T7OjPXtfx_YG5d_Y,29004
 src/pipeline/__init__.py,sha256=BCWl3hwwLkmZc2vQ2fEyUdhis4TPK930HEgXTog8NjA,742
 src/pipeline/asset_payload_server.py,sha256=6LPEcybDk6gKYw6cSVXRKAHmZhkczwkjt3YgE6kN644,9773
 src/pipeline/content_provider.py,sha256=XiSOED7Lw0S663ybZo4iveQRceg5f51tdU9tyv2erSg,1078
@@ -93,13 +93,13 @@ src/pipeline/worker_pool.py,sha256=Yjr_2LXxV2nYNcXNN4gFRPK9KN15GP_IkkAzaplKaTc,1
 src/sources/__init__.py,sha256=wYksO_0IcvlGXLouWapKyqzL0jG0El4rDDGRfwa4Zqk,3350
 src/sources/asset_metadata.py,sha256=OnHwnUSlT1Ow_6Rz2SnJvZyJvaz00TsuQgAnXjTXsdA,6143
 src/sources/atlassian_common.py,sha256=kKriT1BdPwFq4PjqSm5BCb6UE5JaJckHQGObPyk95oM,11982
-src/sources/base.py,sha256=e1PhMOcs1STN1SeQD9RWQ8kNkml6o_njw5WEWKZNg_4,36927
+src/sources/base.py,sha256=Brj7h9L_T9dQ6N6AyY0DrDD_C7dDKYUEmmjc8_SQOuk,42206
 src/sources/dependencies.py,sha256=ei5P3S1CxnNN-zKlChKwzyWEeAApYggPM_YoqrSNWaU,2699
 src/sources/lakehouse_base.py,sha256=c8xj5zzSw6GP_vWMT7aIZNYlsL3t2hkPJWzhYRQbWDc,17217
 src/sources/recipe_normalizer.py,sha256=tgZj_jYnT-67rlDiEF9AsQUS7EcxCvDUUsk2LO8RrRE,6207
 src/sources/s3_client.py,sha256=_v-JlCihlltbbI6dA-OeW4rAvglCKVmj4r8Ib1ViU_A,2368
 src/sources/search_engine_base.py,sha256=meQbpV2Si2T8tBe2DS7VvvpZZ7KVCn6cNqCYCwdwLUE,18308
-src/sources/tabular_base.py,sha256=wDAe-ydHnTQ1LKtlXstBUTjyNjmxmSfN_ZVgy37gLZU,55941
+src/sources/tabular_base.py,sha256=C4CPq1Wz3a78uPRSF634EAVXUM9IGpF36_9-g0_DEUA,57586
 src/sources/tabular_utils.py,sha256=-kiULIyWbNELx9vhQF4O-xbEgkQurg3hGRLCnHHRjbs,7308
 src/sources/azure_blob_storage/__init__.py,sha256=EAsp7N2Vpzc_S-ownm3-ieXVWZSIVXPPNiSlDGCyenI,81
 src/sources/azure_blob_storage/source.py,sha256=PGKY-OEyO_AywVok36M6-Q8Whq2_yGIoRr7m-qNrtus,7144
@@ -108,9 +108,9 @@ src/sources/confluence/source.py,sha256=ngY_RtkHD0pLZbQokPUp580kOnCIl0c-3v_AJmW9
 src/sources/custom/__init__.py,sha256=LeFb8IS8gGSDVZ710S0GcmDeau31fvPHg3GroIzuU_4,61
 src/sources/custom/env.py,sha256=bO82OJVRQ5x8a1AK6jYh5UXS7CG9qodWrxxX6H9xPvU,2543
 src/sources/custom/runner.py,sha256=OSneHmlCNFkl2XNYaNx7X2s_Pt2l11-3lAmfcdAFMKg,15611
-src/sources/custom/source.py,sha256=hqBN5Tu9D5AT4Lm4qNfq0ctq6VT_a69Y8m49oIZ0GjM,50912
+src/sources/custom/source.py,sha256=uHjY18mCBFrXrVI34T4d-l3YL9C9zE0p7uM_by6CFiQ,50925
 src/sources/databricks/__init__.py,sha256=yZZxIF-qhpt_W_zksFQoJGBqaIA40vQCyYLFSksGRFM,69
-src/sources/databricks/source.py,sha256=GfOOldIOIcGr9wRD8Qr9x6xxV8LIG8mJhbOOmdII2RQ,44161
+src/sources/databricks/source.py,sha256=323oLatmMfq2-e3FKz7-VqPNB_lFLBZiai-9HS0S1yw,52048
 src/sources/delta_lake/__init__.py,sha256=KWZQy0l3sT5d7mfeDEbKlbOY1npMhC80dkQW6Ft42Bc,67
 src/sources/delta_lake/source.py,sha256=3RUn0sNULDmN9GStApnJxxrz04AOSHhAaLaP6TAXwjk,5466
 src/sources/dremio/__init__.py,sha256=qIDDLY7WK1lMIU2hGbLxRE_uPLN8-Ksb0FHuwmBZTlw,61
@@ -130,7 +130,7 @@ src/sources/google_cloud_storage/source.py,sha256=yyTh005smphxYdDigSIVQsqxf7qR9u
 src/sources/google_workspace/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 src/sources/google_workspace/source.py,sha256=aHrptTVVGSWE1MyAp37BDwZdU6zWivpZdJgDWnmEou8,36007
 src/sources/hive/__init__.py,sha256=AYWKwROWOgaMDdRTakg5g3vqQqjECKXiozADPr4MeSM,57
-src/sources/hive/source.py,sha256=uv2_-6fBnHU8wEc-a04QRY4N_yzknxVBlC5WiqAodiY,14940
+src/sources/hive/source.py,sha256=Ls9uAugR91t-U13hjtv0uj7xJE0VY-n-BhxYqjmpW-g,15017
 src/sources/hugging_face/__init__.py,sha256=m0E030elrR2LcpIKEv6OBGDAh4BR5HjLrRVpVNB6WRE,71
 src/sources/hugging_face/source.py,sha256=F81pc5cEE56s2xEStfqYrY7tRbPcpiwzcSiqB3Shggg,24802
 src/sources/iceberg/__init__.py,sha256=Wz3kPUaNMSgMzvP9jFxSIbhijXLE1is8G82Pzxtarb4,63
@@ -147,13 +147,13 @@ src/sources/meilisearch/source.py,sha256=O2FYVnPYginrDlAxB10Ua5IAjWeOIjjuH93wL0Z
 src/sources/microsoft_365/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 src/sources/microsoft_365/source.py,sha256=RUUK_EJMzHWs-Y_pebeehXA6g58YpFm-yS9hAat7ntg,39907
 src/sources/mongodb/__init__.py,sha256=IcSTuVMB3JOXAjYpwFcFvRuc2a7m6xgZ1VG5AfM4eJU,63
-src/sources/mongodb/source.py,sha256=klieVNqzxn4-D_PPgUsAtIc5A6Vas8l4K03X9m2x8aA,25992
+src/sources/mongodb/source.py,sha256=BBHPKqNb29SHvjiFPpaFPaFojx_oc5H5vsCZD6R2cQk,26068
 src/sources/mssql/__init__.py,sha256=Y2WuywS2C8lIsiwbikGKQQ9S3Pbp4X5cHyXiDNz8JPI,59
 src/sources/mssql/source.py,sha256=NpsKGZSjoENEvekNZ5BW2WJz6d2DAR24Qp3RsPN_wVQ,26572
 src/sources/mysql/__init__.py,sha256=V0hGtrFJLjGpc3iXzt2k8fH0WzAnyuuDG_2FogcwJBE,59
-src/sources/mysql/source.py,sha256=vHeV4fJ6W78xS5y6hCphbgG4QPAvVmSP1167qOxYDbI,16335
+src/sources/mysql/source.py,sha256=r-VtJeGHjGCTimK3ZcCC9iXHETeaYm_F7XmyhebKHIU,16351
 src/sources/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-src/sources/neo4j/source.py,sha256=UYTVnmvrhterM21498V23qmOVMNdasgTyzWpmDF5Ux4,20208
+src/sources/neo4j/source.py,sha256=Fp2qXNJcMrG5TobzF1qNeLRK0ExoDH056Pd-nZ4LMzA,20334
 src/sources/notion/__init__.py,sha256=FIOPK4itaRGa449jBpvRQd_qlJPPRsjIhSoUBUnMem0,61
 src/sources/notion/client.py,sha256=Wu-JSXI9KnDgfRUzzP8Vm-_TbhY2FDHdEID_Z-EcMd8,8262
 src/sources/notion/source.py,sha256=vaqWGVDtdlBAbuhI2vBf3zRQIG56-9C7_yBuflM1DS8,38218
@@ -205,14 +205,14 @@ src/utils/resources.py,sha256=2ejFjz9r_Kv_d-sUGD0oj54U-ZIUSxDAu7aD4YEv4FQ,2630
 src/utils/sampling_cursor.py,sha256=J1k91Js9ks3mp3JMOb0Jcy5XOkWQlvLDBlxzCXzrNfg,1864
 src/utils/sampling_frontier.py,sha256=4e8Ugy5NiPaEr2YGpQ5-i_Dy3m8J_aOSl81HlFTX_PU,25232
 src/utils/source_files.py,sha256=OKZSW64cur0HTuWPRkV1sJg_Z2LFGWzUsC853gGlFBY,5033
-src/utils/sql_lineage.py,sha256=r9ySlFs-2RpGDHcfAplBu-JHcYRyb6v9xj60ahvtVf0,21465
+src/utils/sql_lineage.py,sha256=ssthZV7W9x-mjQuOybRpX64s7NFnwyubR-0mizRKm0s,21980
 src/utils/torch_runtime.py,sha256=79NYfvJ7fjHGgeLXX8rK63qJFuO2Gao1jjRAQ4nicrk,2527
 src/utils/transcription.py,sha256=WOitCBNTZ7rrGq41ZGH99XmAIyqxvqmnJHxbK3RP7Rs,16943
 src/utils/urn.py,sha256=3ceITLRB9kkIcFghxA_wQNrcuKDqUuHgmhrxZH2wal0,12161
 src/utils/uv_sync.py,sha256=VZ0TZ7t3_pSjxOb20BxUs2XLVjigXnVqa5y69XWlnE8,9307
 src/utils/validation.py,sha256=5NHsgyZwyEONRu9_ADxUAQNPvbMnpMhKL-hpgFEVMm8,3837
 src/utils/video_processing.py,sha256=B6Br9ybFEa_ioglJGo0P9Fmp1e2B95C24joAktwrjww,11293
-classifyre_cli-0.7.0.dist-info/METADATA,sha256=EfXnWAQo_q7BkqzxLYz5kHDlqNBV2IHhpX6450fh0rw,5850
-classifyre_cli-0.7.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-classifyre_cli-0.7.0.dist-info/entry_points.txt,sha256=edxllyMiIFfajgCZ3dTRACXYGC34ZX1AYTezMSR5E_8,45
-classifyre_cli-0.7.0.dist-info/RECORD,,
+classifyre_cli-0.7.2.dist-info/METADATA,sha256=qXrUZKt1K1yD5-Ki51dI9EMooHoOJ9CgWw88LQyNlwo,5850
+classifyre_cli-0.7.2.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+classifyre_cli-0.7.2.dist-info/entry_points.txt,sha256=edxllyMiIFfajgCZ3dTRACXYGC34ZX1AYTezMSR5E_8,45
+classifyre_cli-0.7.2.dist-info/RECORD,,

src/graph/edges.py

@@ -131,6 +131,7 @@ class FieldTransform(StrEnum):
     IDENTITY = "IDENTITY"  # copied through unchanged
     TRANSFORMED = "TRANSFORMED"  # an expression combined the upstreams
     AGGREGATED = "AGGREGATED"  # SUM/COUNT/... over a group
+    MAPPED = "MAPPED"  # the platform says it feeds this column, not how
     INDIRECT = "INDIRECT"  # influenced the rows, not the values
 
 

src/main.py

@@ -971,6 +971,14 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                             source.partial_coverage_reason or "no reason given",
                         )
 
+                    if getattr(source, "listing_complete", False) and hasattr(
+                        sink, "set_listing_complete"
+                    ):
+                        sink.set_listing_complete()
+
+                    if hasattr(sink, "set_sweep_complete"):
+                        sink.set_sweep_complete(getattr(source, "sweep_complete", None))
+
                     # Relationships go *before* finish(): finish() finalizes the
                     # run and marks it COMPLETED, and edges that land after that
                     # belong to a run that already claimed to be done.

src/outputs/rest.py

@@ -203,6 +203,10 @@ class FinalizeIngestRunRequest(BaseModel):
     # "This run covered a slice of the source." Suppresses retirement: absence
     # from a cohort run is not evidence that an asset is gone.
     partial_coverage: bool | None = Field(None, serialization_alias="partialCoverage")
+    discovery_complete: bool | None = Field(None, serialization_alias="discoveryComplete")
+    # Whether every unit of an AUTOMATIC sweep that wraps has been covered once.
+    # Only sent by a run with a wrapping unit; see BaseSource.sweep_complete.
+    sweep_complete: bool | None = Field(None, serialization_alias="sweepComplete")
     # Per ctx.cohort(): keys each band visited, and the weights it ran with.
     cohort_stats: dict[str, Any] | None = Field(None, serialization_alias="cohortStats")
     # Detectors the circuit breaker disabled mid-run (scan-cache identity
@@ -261,6 +265,8 @@ class RestOutputSink:
         self._scan_cache_savings: tuple[int, int] | None = None
         self._partial_coverage: bool = False
         self._partial_coverage_reason: str = ""
+        self._listing_complete: bool = False
+        self._sweep_complete: bool | None = None
         self._detectors_degraded: list[str] = []
         self._cohort_stats: dict[str, Any] | None = None
         self.relationship_report = RelationshipReport()
@@ -279,6 +285,14 @@ class RestOutputSink:
         if reason:
             self._partial_coverage_reason = str(reason)
 
+    def set_listing_complete(self) -> None:
+        """Mark that discovery named every object of the configured scope."""
+        self._listing_complete = True
+
+    def set_sweep_complete(self, complete: bool | None) -> None:
+        """Record whether the wrapping sweep has covered all its ground once."""
+        self._sweep_complete = complete
+
     def set_detectors_degraded(self, keys: list[str]) -> None:
         """Record detectors the breaker disabled mid-run, for the next run's sweep."""
         seen: set[str] = set(self._detectors_degraded)
@@ -415,6 +429,8 @@ class RestOutputSink:
             relationships_dropped=lineage.dropped,
             relationship_errors=lineage.errors or None,
             partial_coverage=True if self._partial_coverage else None,
+            discovery_complete=True if self._listing_complete else None,
+            sweep_complete=self._sweep_complete,
             cohort_stats=self._cohort_stats,
             detectors_degraded=self._detectors_degraded or None,
         )

src/sources/base.py

@@ -89,6 +89,12 @@ class BaseSource(ABC):
         self._next_sampling_cursor: dict[str, Any] | None = None
         self._partial_coverage: bool = False
         self._partial_coverage_reason: str = ""
+        self._listing_complete: bool = False
+        self._listing_errors: int = 0
+        # Ground an AUTOMATIC sweep has to cover, by cursor key, and whether
+        # each unit has been covered once (see ``sweep_complete``).
+        self._sweep_units: dict[str, bool] = {}
+        self._sweep_wraps: bool = False
         self._sampling_cursor_lock = threading.Lock()
         self._apply_initial_sampling_override(normalized_recipe)
         recipe.clear()
@@ -177,6 +183,72 @@ class BaseSource(ABC):
         """
         return self._next_sampling_cursor
 
+    # ── Has the sweep gone round once? ───────────────────────────────────
+    #
+    # A cursor that wraps (an offset that returns to 0 at the end, a walk that
+    # starts over) never stops moving, by design: walking again is the only way
+    # a store that cannot be ordered by age shows its new rows. So "the cursor
+    # moved" cannot tell a sweep that is still reaching new ground from one that
+    # is going round for the second time, and a scheduler that reads it that way
+    # re-runs such a source every minute for good.
+    #
+    # Each unit of ground therefore records, once, that it has been covered: a
+    # frontier says so itself (``done``); a wrapping unit says so by wrapping
+    # (kept in the cursor under ``LAPPED_KEY``, which only ever grows). The run
+    # reports ``sweep_complete`` when every unit it touched has been covered and
+    # at least one of them wraps -- the only case the cursor cannot speak for.
+
+    #: Cursor key listing the wrapping units that have been covered once.
+    LAPPED_KEY = "_lapped"
+
+    def _lapped_keys(self) -> set[str]:
+        """Wrapping units known to have been covered once, as of this run.
+
+        Callers hold ``_sampling_cursor_lock`` when they mean to write back.
+        """
+        keys: set[str] = set()
+        for cursor in (self._sampling_cursor, self._next_sampling_cursor):
+            raw = cursor.get(self.LAPPED_KEY) if isinstance(cursor, dict) else None
+            if isinstance(raw, list):
+                keys.update(str(k) for k in raw)
+        return keys
+
+    def note_sweep_frontier(self, key: str, *, done: bool) -> None:
+        """Record whether the frontier under ``key`` has covered its ground."""
+        self._sweep_units[key] = bool(done)
+
+    def note_sweep_wrap(self, key: str, *, wrapped: bool) -> None:
+        """Record that the wrapping unit ``key`` was read this run.
+
+        ``wrapped`` says this run reached its end and starts over. That is
+        remembered in the cursor from then on, so a later run that is
+        mid-walk still counts the unit as covered.
+        """
+        with self._sampling_cursor_lock:
+            self._sweep_wraps = True
+            lapped = self._lapped_keys()
+            if wrapped and key not in lapped:
+                lapped.add(key)
+                nxt = (
+                    self._next_sampling_cursor
+                    if isinstance(self._next_sampling_cursor, dict)
+                    else dict(self._sampling_cursor)
+                )
+                self._next_sampling_cursor = {**nxt, self.LAPPED_KEY: sorted(lapped)}
+            self._sweep_units[key] = key in lapped
+
+    @property
+    def sweep_complete(self) -> bool | None:
+        """Whether every unit this run read has been covered at least once.
+
+        None when the run has nothing to say: no unit of it wraps, so the
+        cursor itself already tells a finished sweep (it stops moving).
+        """
+        units = getattr(self, "_sweep_units", None)
+        if not getattr(self, "_sweep_wraps", False) or not units:
+            return None
+        return all(units.values())
+
     def declare_partial_coverage(self, reason: str = "") -> None:
         """Say that this run looked at only part of the source.
 
@@ -191,6 +263,34 @@ class BaseSource(ABC):
         if reason and not self._partial_coverage_reason:
             self._partial_coverage_reason = str(reason)
 
+    def note_listing_error(self) -> None:
+        """Record that part of the catalog could not be listed this run.
+
+        A listing that skipped a schema, a folder or a label cannot say that
+        what it did not list is gone, so it forfeits :meth:`declare_listing_complete`.
+        """
+        self._listing_errors = getattr(self, "_listing_errors", 0) + 1
+
+    def declare_listing_complete(self) -> None:
+        """Say that discovery named every object of the configured scope.
+
+        Sampling decides how much of each asset is read, not which assets
+        exist. A catalog connector (tables, node labels) lists its whole scope
+        on every run whatever the strategy, so an asset missing from that list
+        is gone and can be retired -- without this, only strategy=ALL ever
+        retired anything and every other source kept deleted tables, moved
+        databases and renamed labels as "unchanged" forever.
+
+        Ignored when any part of the listing failed or the run declared partial
+        coverage.
+        """
+        if getattr(self, "_listing_errors", 0) == 0:
+            self._listing_complete = True
+
+    @property
+    def listing_complete(self) -> bool:
+        return bool(getattr(self, "_listing_complete", False)) and not self.partial_coverage
+
     @property
     def partial_coverage(self) -> bool:
         return bool(getattr(self, "_partial_coverage", False))
@@ -231,16 +331,24 @@ class BaseSource(ABC):
         saved = self._sampling_cursor.get(key)
         return saved if isinstance(saved, int) and not isinstance(saved, bool) and saved >= 0 else 0
 
-    def record_automatic_offset(self, key: str, *, prev_offset: int, fetched: int) -> None:
+    def record_automatic_offset(
+        self, key: str, *, prev_offset: int, fetched: int, track: bool = True
+    ) -> None:
         """Advance a keyed offset cursor; wrap to 0 once a page underfills.
 
         For a store that can only be paged by position (``skip``/``OFFSET``)
         and offers nothing to order by, so new rows cannot be told from old
         ones. Wrapping is how such a source ever sees them.
+
+        ``track=False`` keeps the unit out of ``sweep_complete``, for a caller
+        whose cursor holds state this offset knows nothing about.
         """
         size = self.sampling_window_size()
-        next_offset = 0 if fetched < size else prev_offset + fetched
+        wrapped = fetched < size
+        next_offset = 0 if wrapped else prev_offset + fetched
         self._record_cursor_key(key, next_offset)
+        if track:
+            self.note_sweep_wrap(key, wrapped=wrapped)
 
     def automatic_window(
         self,
@@ -314,7 +422,9 @@ class BaseSource(ABC):
         )
         window = items[offset : offset + size]
         next_offset = offset + len(window)
-        self._record_cursor_key(key, 0 if next_offset >= len(items) else next_offset)
+        wrapped = next_offset >= len(items)
+        self._record_cursor_key(key, 0 if wrapped else next_offset)
+        self.note_sweep_wrap(key, wrapped=wrapped)
         return [item for _, item in window]
 
     def automatic_ranged_window(
@@ -381,6 +491,7 @@ class BaseSource(ABC):
         state = frontier.to_state()
         if state is not None:
             self._record_cursor_key(key, state)
+        self.note_sweep_frontier(key, done=frontier.done or frontier.fresh)
         return window
 
     def automatic_frontier(self, key: str) -> ListFrontier:
@@ -396,6 +507,7 @@ class BaseSource(ABC):
         state = frontier.to_state()
         if state is not None:
             self._record_cursor_key(key, state)
+        self.note_sweep_frontier(key, done=frontier.done or frontier.fresh)
 
     @staticmethod
     def _read_bool_env(name: str) -> bool | None:

src/sources/custom/source.py

@@ -622,7 +622,7 @@ class CustomSource(BaseSource):
         # P15). The offset is merged in, never substituted.
         self.set_next_sampling_cursor({k: v for k, v in self._sampling_cursor.items() if k != key})
         self.record_automatic_offset(
-            key, prev_offset=offset, fetched=int(frame.get("produced") or 0)
+            key, prev_offset=offset, fetched=int(frame.get("produced") or 0), track=False
         )
 
     def _handled_offset(self, frame: dict[str, Any]) -> bool:

src/sources/databricks/source.py

@@ -7,7 +7,7 @@ from collections.abc import AsyncGenerator, Generator
 from contextlib import closing
 from dataclasses import dataclass
 from datetime import UTC, datetime, timedelta
-from typing import Any
+from typing import Any, ClassVar
 from urllib.parse import urlparse
 
 import requests
@@ -19,6 +19,7 @@ from ...graph.edges import (
     Method,
     Ref,
     flow,
+    same_as,
 )
 from ...models.generated_input import (
     AzureServicePrincipal,
@@ -68,6 +69,16 @@ class PipelineRef:
     state: str | None
 
 
+@dataclass(frozen=True)
+class _ForeignSystem:
+    """The database behind a foreign catalog, as its own connector names it."""
+
+    platform: str
+    authority: str
+    database: str | None
+    connection: str
+
+
 def _quote_identifier(identifier: str) -> str:
     return f"`{identifier.replace('`', '``')}`"
 
@@ -109,6 +120,10 @@ class DatabricksSource(BaseTabularSource):
         self._validate_auth_configuration()
 
         self.session = requests.Session()
+        # table_key -> upstream table -> ids of the notebooks Unity Catalog saw
+        # move data between the two. Filled by _lineage_refs_for_table.
+        self._lineage_notebook_ids: dict[tuple[str, ...], dict[tuple[str, str, str], set[str]]] = {}
+        self._foreign_systems: dict[str, _ForeignSystem | None] = {}
         self._access_token: str | None = None
         self._access_token_expiry: datetime | None = None
 
@@ -409,9 +424,13 @@ class DatabricksSource(BaseTabularSource):
         without_scheme = raw.split("://", 1)[-1]
         return without_scheme.strip("/").split("/", 1)[0]
 
-    def _available_columns(self, table_ref: TableRef) -> list[str]:
+    def _available_column_types(self, table_ref: TableRef) -> dict[str, str]:
+        # ``system.information_schema`` sees every catalog from one connection;
+        # the default query reads only the connection's current catalog and a
+        # parameter style Databricks does not take, so types used to come back
+        # empty for every column.
         query = (
-            "SELECT column_name "
+            "SELECT column_name, COALESCE(full_data_type, data_type) "
             "FROM system.information_schema.columns "
             f"WHERE table_catalog = {_quote_literal(table_ref.database)} "
             f"AND table_schema = {_quote_literal(table_ref.schema or '')} "
@@ -421,17 +440,17 @@ class DatabricksSource(BaseTabularSource):
         conn = self._get_cached_connection(table_ref.database)
         with conn.cursor() as cursor:
             cursor.execute(query)
-            columns: list[str] = []
+            columns: dict[str, str] = {}
             for row in cursor.fetchall():
-                candidate: Any | None = None
-                try:
-                    candidate = row[0]  # type: ignore[index]
-                except Exception:
-                    candidate = None
-                if isinstance(candidate, str):
-                    columns.append(candidate)
+                name = row[0] if len(row) > 0 else None
+                if isinstance(name, str):
+                    data_type = row[1] if len(row) > 1 else None
+                    columns[name] = str(data_type) if data_type is not None else ""
             return columns
 
+    def _available_columns(self, table_ref: TableRef) -> list[str]:
+        return list(self._available_column_types(table_ref))
+
     # ── Databricks-specific cell serialization ───────────────────────────
 
     def _serialize_cell(self, value: Any) -> str:
@@ -598,6 +617,7 @@ class DatabricksSource(BaseTabularSource):
         try:
             schemas = self._list_schemas_for_catalog(database)
         except Exception as exc:
+            self.note_listing_error()
             logger.warning("Skipping catalog %s due to schema listing error: %s", database, exc)
             return tables
         for schema in schemas:
@@ -606,6 +626,7 @@ class DatabricksSource(BaseTabularSource):
             try:
                 tables.extend(self._list_tables_for_schema(database, schema))
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning(
                     "Skipping schema %s.%s due to table listing error: %s",
                     database,
@@ -653,14 +674,30 @@ class DatabricksSource(BaseTabularSource):
             },
         )
         refs: set[tuple[str, str, str]] = set()
+        notebooks: dict[tuple[str, str, str], set[str]] = {}
         for key in ("upstreams", "upstream_tables"):
             entries = response.get(key)
             if isinstance(entries, list):
                 for entry in entries:
                     if isinstance(entry, dict):
+                        if (entry.get("tableInfo") or {}).get("has_permission") is False:
+                            # Unity Catalog knows the upstream and will not name
+                            # it to this identity, so the table looks sourceless.
+                            logger.warning(
+                                "%s has an upstream this identity may not see; "
+                                "grant it BROWSE on that catalog",
+                                table_ref.display_name,
+                            )
+                            continue
                         parsed = self._lineage_table_ref_from_payload(entry)
                         if parsed:
                             refs.add(parsed)
+                            for info in entry.get("notebookInfos") or []:
+                                if isinstance(info, dict) and info.get("notebook_id") is not None:
+                                    notebooks.setdefault(parsed, set()).add(
+                                        str(info["notebook_id"])
+                                    )
+        self._lineage_notebook_ids[table_ref.table_key] = notebooks
         return refs
 
     def _emit_table_lineage(
@@ -668,6 +705,7 @@ class DatabricksSource(BaseTabularSource):
         table_ref: TableRef,
         upstream_refs: set[tuple[str, str, str]],
         table_hash_by_key: dict[tuple[str, ...], str],
+        notebook_hash_by_id: dict[str, str] | None = None,
     ) -> None:
         """Turn Unity Catalog's lineage into typed edges.
 
@@ -675,6 +713,9 @@ class DatabricksSource(BaseTabularSource):
         question directly, so nothing here is parsed out of SQL — these edges
         are as trustworthy as lineage gets, and are marked SYSTEM_CATALOG.
 
+        When Unity Catalog saw a notebook run the statement, the edge carries it
+        as ``via`` -- if that notebook is one of this scan's assets.
+
         The upstreams that are *not* part of this scan are the important half.
         They used to be dropped on the floor (``if target in table_hash_by_key``),
         which meant a table fed by a catalog outside the configured scope looked
@@ -686,23 +727,52 @@ class DatabricksSource(BaseTabularSource):
             return
 
         column_mappings = self._column_mappings_for_table(table_ref)
+        notebook_ids = self._lineage_notebook_ids.get(table_ref.table_key, {})
         for upstream in sorted(upstream_refs):
             upstream_hash = table_hash_by_key.get(upstream)
-            upstream_ref = (
-                Ref.asset(upstream_hash) if upstream_hash else self._urn_ref_for_key(upstream)
-            )
+            foreign_urn = self._foreign_urn(upstream)
+            upstream_ref: Ref | None
+            if foreign_urn:
+                upstream_ref = Ref.urn(foreign_urn)
+            elif upstream_hash:
+                upstream_ref = Ref.asset(upstream_hash)
+            else:
+                upstream_ref = self._urn_ref_for_key(upstream)
             if upstream_ref is None:
                 continue
+            via_hash = next(
+                (
+                    notebook_hash_by_id[notebook_id]
+                    for notebook_id in sorted(notebook_ids.get(upstream, ()))
+                    if notebook_id in (notebook_hash_by_id or {})
+                ),
+                None,
+            )
             self.add_edge(
                 flow(
                     upstream=upstream_ref,
                     downstream=Ref.asset(downstream_hash),
                     type=FlowType.TRANSFORM,
                     fields=column_mappings.get(upstream, []),
+                    via=Ref.asset(via_hash) if via_hash else None,
                     method=Method.SYSTEM_CATALOG,
+                    evidence=self._foreign_evidence(upstream) if foreign_urn else None,
                 )
             )
 
+    def _emit_foreign_identity(self, table_ref: TableRef, asset_hash: str) -> None:
+        """A scanned table of a foreign catalog is a mirror of the real table."""
+        foreign_urn = self._foreign_urn(table_ref.table_key)
+        if not foreign_urn:
+            return
+        self.add_edge(
+            same_as(
+                Ref.asset(asset_hash),
+                Ref.urn(foreign_urn),
+                evidence=self._foreign_evidence(table_ref.table_key),
+            )
+        )
+
     def _column_mappings_for_table(
         self, table_ref: TableRef
     ) -> dict[tuple[str, str, str], list[FieldMapping]]:
@@ -744,14 +814,12 @@ class DatabricksSource(BaseTabularSource):
 
         return {
             upstream: [
+                # Unity Catalog names the columns and nothing between them, so a
+                # matching name proves nothing: `lower(email) AS email` matches.
                 FieldMapping(
                     downstream=downstream,
                     upstreams=sorted(sources),
-                    type=(
-                        FieldTransform.IDENTITY
-                        if sources == {downstream}
-                        else FieldTransform.TRANSFORMED
-                    ),
+                    type=FieldTransform.MAPPED,
                 )
                 for downstream, sources in sorted(columns.items())
             ]
@@ -767,6 +835,86 @@ class DatabricksSource(BaseTabularSource):
         except UrnError:
             return None
 
+    # ── Foreign catalogs (Lakehouse Federation) ──────────────────────────
+    #
+    # A foreign catalog is Databricks' name for tables that live in another
+    # database. Unity Catalog records lineage through it like any other catalog,
+    # so without this the upstream is `databricks://.../app_db/core/customers`,
+    # a name no PostgreSQL scan will ever write, and the path stops at the
+    # boundary it was most worth crossing.
+
+    #: UC connection type -> (URN platform, catalog option naming the database).
+    #: None where the schema level already is the database, as in MySQL.
+    _FOREIGN_PLATFORMS: ClassVar[dict[str, tuple[str, str | None]]] = {
+        "POSTGRESQL": ("postgres", "database"),
+        "MYSQL": ("mysql", None),
+        "SQLSERVER": ("mssql", "database"),
+        "ORACLE": ("oracle", "service_name"),
+        "SNOWFLAKE": ("snowflake", "database"),
+    }
+
+    def _foreign_system(self, catalog: str) -> _ForeignSystem | None:
+        if catalog in self._foreign_systems:
+            return self._foreign_systems[catalog]
+        system: _ForeignSystem | None = None
+        try:
+            system = self._read_foreign_system(catalog)
+        except Exception as exc:
+            # USE CONNECTION is a separate grant. The edge keeps its Databricks
+            # name, which is true if not useful, rather than the scan failing.
+            logger.warning("Could not read the connection of catalog %s: %s", catalog, exc)
+        self._foreign_systems[catalog] = system
+        return system
+
+    def _read_foreign_system(self, catalog: str) -> _ForeignSystem | None:
+        info = self._request_json("get", f"/api/2.1/unity-catalog/catalogs/{catalog}")
+        connection_name = info.get("connection_name")
+        if info.get("catalog_type") != "FOREIGN_CATALOG" or not connection_name:
+            return None
+        connection = self._request_json(
+            "get", f"/api/2.1/unity-catalog/connections/{connection_name}"
+        )
+        mapping = self._FOREIGN_PLATFORMS.get(str(connection.get("connection_type")))
+        if mapping is None:
+            return None
+        platform, database_option = mapping
+
+        options = connection.get("options") or {}
+        host = str(options.get("host") or "").strip()
+        if not host:
+            # A URN scoped to the wrong host would stitch two unrelated systems.
+            return None
+        if platform == "snowflake":
+            authority = host.removesuffix(".snowflakecomputing.com")
+        else:
+            port = options.get("port")
+            authority = f"{host}:{port}" if port else host
+
+        database = None
+        if database_option:
+            database = str((info.get("options") or {}).get(database_option) or "").strip()
+            if not database:
+                return None
+        return _ForeignSystem(platform, authority, database, str(connection_name))
+
+    def _foreign_urn(self, table_key: tuple[str, ...]) -> str | None:
+        """The URN the underlying system's own connector writes for this table."""
+        system = self._foreign_system(table_key[0])
+        if system is None:
+            return None
+        name = (system.database, *table_key[1:]) if system.database else table_key[1:]
+        try:
+            return str(Urn.of(system.platform, system.authority, *name))
+        except UrnError:
+            return None
+
+    def _foreign_evidence(self, table_key: tuple[str, ...]) -> dict[str, Any]:
+        system = self._foreign_system(table_key[0])
+        return {
+            "foreign_catalog": table_key[0],
+            "connection": system.connection if system else None,
+        }
+
     # ── Notebooks ────────────────────────────────────────────────────────
 
     def _iter_notebooks(self) -> Generator[NotebookRef, None, None]:
@@ -786,6 +934,7 @@ class DatabricksSource(BaseTabularSource):
                     "get", "/api/2.0/workspace/list", params={"path": path}
                 )
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning("Skipping workspace path %s due to listing error: %s", path, exc)
                 continue
             objects = payload.get("objects")
@@ -865,6 +1014,7 @@ class DatabricksSource(BaseTabularSource):
             try:
                 payload = self._request_json("get", "/api/2.0/pipelines", params=params)
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning("Could not list Databricks pipelines: %s", exc)
                 break
             values: list[dict[str, Any]] = []
@@ -935,6 +1085,15 @@ class DatabricksSource(BaseTabularSource):
             t.table_key: self.generate_hash_id(t.raw_id) for t in tables
         }
 
+        # Listed before the tables so a lineage edge can name the notebook that
+        # produced it; the notebook assets are still emitted after the tables.
+        notebooks = list(self._iter_notebooks())
+        notebook_hash_by_id = {
+            nb.object_id: self.generate_hash_id(f"notebook_#_{nb.path}")
+            for nb in notebooks
+            if nb.object_id
+        }
+
         include_lineage = self._extraction_options().include_table_lineage
         if include_lineage and tables:
             logger.info("Fetching table lineage for %d table(s)...", len(tables))
@@ -957,7 +1116,9 @@ class DatabricksSource(BaseTabularSource):
                         for target in sorted(upstream_refs)
                         if target in table_hash_by_key
                     ]
-                    self._emit_table_lineage(table_ref, upstream_refs, table_hash_by_key)
+                    self._emit_table_lineage(
+                        table_ref, upstream_refs, table_hash_by_key, notebook_hash_by_id
+                    )
                 except Exception as exc:
                     logger.warning(
                         "Could not resolve Databricks lineage for %s: %s", table_label, exc
@@ -967,6 +1128,8 @@ class DatabricksSource(BaseTabularSource):
             self._table_lookup[asset.hash] = table_ref
             batch.append(asset)
             emitted_tables += 1
+            if include_lineage:
+                self._emit_foreign_identity(table_ref, asset.hash)
 
             if len(batch) >= self.BATCH_SIZE:
                 yield batch
@@ -974,7 +1137,7 @@ class DatabricksSource(BaseTabularSource):
 
         # Notebooks
         notebook_count = 0
-        for notebook in self._iter_notebooks():
+        for notebook in notebooks:
             if self._aborted:
                 break
             batch.append(self._notebook_to_asset(notebook))
@@ -997,6 +1160,7 @@ class DatabricksSource(BaseTabularSource):
                 batch = []
         if pipeline_count:
             logger.info("Discovered %d pipeline(s)", pipeline_count)
+        self._declare_listing_if_whole()
 
         if batch:
             yield batch

src/sources/hive/source.py

@@ -293,7 +293,10 @@ class HiveSource(BaseTabularSource):
             upstreams = [
                 key
                 for key in upstream_tables_from_sql(
-                    definition, dialect="hive", default_database=view.database
+                    definition,
+                    dialect="hive",
+                    default_database=view.database,
+                    two_level=True,
                 )
                 if key != view.table_key and key in scoped
             ]

src/sources/mongodb/source.py

@@ -507,6 +507,7 @@ class MongoDBSource(BaseSource):
                 if next_state.get(name) is not None:
                     next_state[name] = self._encode_id(next_state[name])
             self._record_cursor_key(key, next_state)
+        self.note_sweep_frontier(key, done=frontier.done or frontier.fresh)
         return documents
 
     def _sample_collection_documents(self, collection_ref: CollectionRef) -> list[dict[str, Any]]:

src/sources/mysql/source.py

@@ -322,7 +322,7 @@ class MySQLSource(BaseTabularSource):
                 upstreams = [
                     key
                     for key in upstream_tables_from_sql(
-                        definition, dialect="mysql", default_database=database
+                        definition, dialect="mysql", default_database=database, two_level=True
                     )
                     if key != view_key
                 ]

src/sources/neo4j/source.py

@@ -325,6 +325,9 @@ class Neo4jSource(BaseSource):
                 yield batch
                 batch = []
 
+        if not self._aborted and not self._scope_options().node_limit_per_label:
+            self.declare_listing_complete()
+
         if batch:
             logger.info("Emitting final batch of %d asset(s)", len(batch))
             yield batch

src/sources/tabular_base.py

@@ -319,9 +319,23 @@ class BaseTabularSource(BaseSource):
             try:
                 tables.extend(self._list_tables_for_database(database))
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning("Skipping database %s due to listing error: %s", database, exc)
         return tables
 
+    def _listing_is_limited(self) -> bool:
+        """Whether a configured cap can have cut the table list short."""
+        scope_options = getattr(self, "_scope_options", None)
+        scope = scope_options() if callable(scope_options) else None
+        return bool(
+            getattr(scope, "table_limit", None) or getattr(scope, "table_limit_per_schema", None)
+        )
+
+    def _declare_listing_if_whole(self) -> None:
+        """Offer this run's table list as the whole scope, when it plausibly is."""
+        if not self._aborted and not self._listing_is_limited():
+            self.declare_listing_complete()
+
     # ── Primary keys ─────────────────────────────────────────────────────
 
     def _get_primary_key_columns(self, table_ref: TableRef) -> list[str]:
@@ -689,6 +703,18 @@ class BaseTabularSource(BaseSource):
             nxt["tables"] = tables
             self._next_sampling_cursor = nxt
             self._automatic_advanced.add(key)
+            self._note_table_sweep(key, state)
+
+    def _note_table_sweep(self, key: str, state: dict[str, Any] | None) -> None:
+        """Tell the sweep tracker whether this table has been covered once."""
+        if state is None:
+            self.note_sweep_frontier(key, done=True)
+        elif state.get("mode") == self._WRAP_MODE:
+            # Walked and wrapped: covered once the walk has reached the end.
+            self._sweep_wraps = True
+            self._sweep_units[key] = bool(state.get("lapped"))
+        else:
+            self.note_sweep_frontier(key, done=bool(state.get("done")) or not state.get("head"))
 
     def _fetch_page_key_range(
         self,
@@ -836,6 +862,7 @@ class BaseTabularSource(BaseSource):
             )
             # Exhausted (or a key that cannot be stored, e.g. binary) leaves no
             # position, so the next run starts the walk again.
+            exhausted = len(rows) < rows_per_page
             if rows and len(rows) >= rows_per_page:
                 last_row = rows[-1]
                 position = [
@@ -846,8 +873,13 @@ class BaseTabularSource(BaseSource):
         else:
             offset = int(saved.get("offset") or 0)
             rows, column_names = self._fetch_one_page(table_ref, base_query, rows_per_page, offset)
+            exhausted = len(rows) < rows_per_page
             if rows and len(rows) >= rows_per_page:
                 next_state["offset"] = offset + len(rows)
+        # Sticky: once the walk has reached the end, the table counts as
+        # covered however many times it is walked again.
+        if exhausted or saved.get("lapped"):
+            next_state["lapped"] = True
 
         self._record_table_cursor(key, next_state)
 
@@ -1097,6 +1129,7 @@ class BaseTabularSource(BaseSource):
         table_hash_by_key: dict[tuple[str, ...], str] = {
             t.table_key: self.generate_hash_id(t.raw_id) for t in tables
         }
+        self._declare_listing_if_whole()
         fk_links = self._collect_foreign_key_links(tables)
         self._emit_relationship_edges(tables, table_hash_by_key, fk_links)
 

src/utils/sql_lineage.py

@@ -444,6 +444,7 @@ def upstream_tables_from_sql(
     dialect: str | None = None,
     default_database: str | None = None,
     default_schema: str | None = None,
+    two_level: bool = False,
 ) -> list[tuple[str, ...]]:
     """Table keys a statement reads from, for catalogs with no dependency view.
 
@@ -453,6 +454,11 @@ def upstream_tables_from_sql(
     the wrong default schema points at nothing — so callers should prefer the
     catalog wherever it exists and treat these edges as SQL_PARSED.
 
+    ``two_level`` is for engines whose tables are named ``database.table`` (MySQL,
+    Hive). sqlglot files the first part under ``db``, so without it a qualified
+    name came back as ``(default_database, database, table)`` -- a key no asset
+    has, which turned every such view's edges into unresolved ones.
+
     CTE names are excluded: a ``WITH`` alias is not a table, and emitting it
     would create lineage to an object that does not exist.
     """
@@ -481,6 +487,11 @@ def upstream_tables_from_sql(
         name = table.name
         if not name or name.lower() in cte_names:
             continue
+        if two_level:
+            namespace = table.text("db") or default_database
+            if namespace:
+                keys.add((namespace, name))
+            continue
         schema = table.text("db") or default_schema
         database = table.text("catalog") or default_database
         if database and schema: