classifyre-cli 0.6.18__tar.gz → 0.7.1__tar.gz

classifyre_cli-0.7.1/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 325 packages in 329ms
+Checked 52 packages in 4ms

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.6.18
+Version: 0.7.1
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.6.18",
+  "version": "0.7.1",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.6.18"
+version = "0.7.1"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/src/main.py

@@ -971,6 +971,11 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                             source.partial_coverage_reason or "no reason given",
                         )
 
+                    if getattr(source, "listing_complete", False) and hasattr(
+                        sink, "set_listing_complete"
+                    ):
+                        sink.set_listing_complete()
+
                     # Relationships go *before* finish(): finish() finalizes the
                     # run and marks it COMPLETED, and edges that land after that
                     # belong to a run that already claimed to be done.

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/src/outputs/rest.py

@@ -203,6 +203,7 @@ class FinalizeIngestRunRequest(BaseModel):
     # "This run covered a slice of the source." Suppresses retirement: absence
     # from a cohort run is not evidence that an asset is gone.
     partial_coverage: bool | None = Field(None, serialization_alias="partialCoverage")
+    discovery_complete: bool | None = Field(None, serialization_alias="discoveryComplete")
     # Per ctx.cohort(): keys each band visited, and the weights it ran with.
     cohort_stats: dict[str, Any] | None = Field(None, serialization_alias="cohortStats")
     # Detectors the circuit breaker disabled mid-run (scan-cache identity
@@ -261,6 +262,7 @@ class RestOutputSink:
         self._scan_cache_savings: tuple[int, int] | None = None
         self._partial_coverage: bool = False
         self._partial_coverage_reason: str = ""
+        self._listing_complete: bool = False
         self._detectors_degraded: list[str] = []
         self._cohort_stats: dict[str, Any] | None = None
         self.relationship_report = RelationshipReport()
@@ -279,6 +281,10 @@ class RestOutputSink:
         if reason:
             self._partial_coverage_reason = str(reason)
 
+    def set_listing_complete(self) -> None:
+        """Mark that discovery named every object of the configured scope."""
+        self._listing_complete = True
+
     def set_detectors_degraded(self, keys: list[str]) -> None:
         """Record detectors the breaker disabled mid-run, for the next run's sweep."""
         seen: set[str] = set(self._detectors_degraded)
@@ -415,6 +421,7 @@ class RestOutputSink:
             relationships_dropped=lineage.dropped,
             relationship_errors=lineage.errors or None,
             partial_coverage=True if self._partial_coverage else None,
+            discovery_complete=True if self._listing_complete else None,
             cohort_stats=self._cohort_stats,
             detectors_degraded=self._detectors_degraded or None,
         )

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/src/sources/base.py

@@ -89,6 +89,8 @@ class BaseSource(ABC):
         self._next_sampling_cursor: dict[str, Any] | None = None
         self._partial_coverage: bool = False
         self._partial_coverage_reason: str = ""
+        self._listing_complete: bool = False
+        self._listing_errors: int = 0
         self._sampling_cursor_lock = threading.Lock()
         self._apply_initial_sampling_override(normalized_recipe)
         recipe.clear()
@@ -191,6 +193,34 @@ class BaseSource(ABC):
         if reason and not self._partial_coverage_reason:
             self._partial_coverage_reason = str(reason)
 
+    def note_listing_error(self) -> None:
+        """Record that part of the catalog could not be listed this run.
+
+        A listing that skipped a schema, a folder or a label cannot say that
+        what it did not list is gone, so it forfeits :meth:`declare_listing_complete`.
+        """
+        self._listing_errors = getattr(self, "_listing_errors", 0) + 1
+
+    def declare_listing_complete(self) -> None:
+        """Say that discovery named every object of the configured scope.
+
+        Sampling decides how much of each asset is read, not which assets
+        exist. A catalog connector (tables, node labels) lists its whole scope
+        on every run whatever the strategy, so an asset missing from that list
+        is gone and can be retired -- without this, only strategy=ALL ever
+        retired anything and every other source kept deleted tables, moved
+        databases and renamed labels as "unchanged" forever.
+
+        Ignored when any part of the listing failed or the run declared partial
+        coverage.
+        """
+        if getattr(self, "_listing_errors", 0) == 0:
+            self._listing_complete = True
+
+    @property
+    def listing_complete(self) -> bool:
+        return bool(getattr(self, "_listing_complete", False)) and not self.partial_coverage
+
     @property
     def partial_coverage(self) -> bool:
         return bool(getattr(self, "_partial_coverage", False))

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/src/sources/databricks/source.py

@@ -109,6 +109,9 @@ class DatabricksSource(BaseTabularSource):
         self._validate_auth_configuration()
 
         self.session = requests.Session()
+        # table_key -> upstream table -> ids of the notebooks Unity Catalog saw
+        # move data between the two. Filled by _lineage_refs_for_table.
+        self._lineage_notebook_ids: dict[tuple[str, ...], dict[tuple[str, str, str], set[str]]] = {}
         self._access_token: str | None = None
         self._access_token_expiry: datetime | None = None
 
@@ -409,9 +412,13 @@ class DatabricksSource(BaseTabularSource):
         without_scheme = raw.split("://", 1)[-1]
         return without_scheme.strip("/").split("/", 1)[0]
 
-    def _available_columns(self, table_ref: TableRef) -> list[str]:
+    def _available_column_types(self, table_ref: TableRef) -> dict[str, str]:
+        # ``system.information_schema`` sees every catalog from one connection;
+        # the default query reads only the connection's current catalog and a
+        # parameter style Databricks does not take, so types used to come back
+        # empty for every column.
         query = (
-            "SELECT column_name "
+            "SELECT column_name, COALESCE(full_data_type, data_type) "
             "FROM system.information_schema.columns "
             f"WHERE table_catalog = {_quote_literal(table_ref.database)} "
             f"AND table_schema = {_quote_literal(table_ref.schema or '')} "
@@ -421,17 +428,17 @@ class DatabricksSource(BaseTabularSource):
         conn = self._get_cached_connection(table_ref.database)
         with conn.cursor() as cursor:
             cursor.execute(query)
-            columns: list[str] = []
+            columns: dict[str, str] = {}
             for row in cursor.fetchall():
-                candidate: Any | None = None
-                try:
-                    candidate = row[0]  # type: ignore[index]
-                except Exception:
-                    candidate = None
-                if isinstance(candidate, str):
-                    columns.append(candidate)
+                name = row[0] if len(row) > 0 else None
+                if isinstance(name, str):
+                    data_type = row[1] if len(row) > 1 else None
+                    columns[name] = str(data_type) if data_type is not None else ""
             return columns
 
+    def _available_columns(self, table_ref: TableRef) -> list[str]:
+        return list(self._available_column_types(table_ref))
+
     # ── Databricks-specific cell serialization ───────────────────────────
 
     def _serialize_cell(self, value: Any) -> str:
@@ -598,6 +605,7 @@ class DatabricksSource(BaseTabularSource):
         try:
             schemas = self._list_schemas_for_catalog(database)
         except Exception as exc:
+            self.note_listing_error()
             logger.warning("Skipping catalog %s due to schema listing error: %s", database, exc)
             return tables
         for schema in schemas:
@@ -606,6 +614,7 @@ class DatabricksSource(BaseTabularSource):
             try:
                 tables.extend(self._list_tables_for_schema(database, schema))
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning(
                     "Skipping schema %s.%s due to table listing error: %s",
                     database,
@@ -653,6 +662,7 @@ class DatabricksSource(BaseTabularSource):
             },
         )
         refs: set[tuple[str, str, str]] = set()
+        notebooks: dict[tuple[str, str, str], set[str]] = {}
         for key in ("upstreams", "upstream_tables"):
             entries = response.get(key)
             if isinstance(entries, list):
@@ -661,6 +671,12 @@ class DatabricksSource(BaseTabularSource):
                         parsed = self._lineage_table_ref_from_payload(entry)
                         if parsed:
                             refs.add(parsed)
+                            for info in entry.get("notebookInfos") or []:
+                                if isinstance(info, dict) and info.get("notebook_id") is not None:
+                                    notebooks.setdefault(parsed, set()).add(
+                                        str(info["notebook_id"])
+                                    )
+        self._lineage_notebook_ids[table_ref.table_key] = notebooks
         return refs
 
     def _emit_table_lineage(
@@ -668,6 +684,7 @@ class DatabricksSource(BaseTabularSource):
         table_ref: TableRef,
         upstream_refs: set[tuple[str, str, str]],
         table_hash_by_key: dict[tuple[str, ...], str],
+        notebook_hash_by_id: dict[str, str] | None = None,
     ) -> None:
         """Turn Unity Catalog's lineage into typed edges.
 
@@ -675,6 +692,9 @@ class DatabricksSource(BaseTabularSource):
         question directly, so nothing here is parsed out of SQL — these edges
         are as trustworthy as lineage gets, and are marked SYSTEM_CATALOG.
 
+        When Unity Catalog saw a notebook run the statement, the edge carries it
+        as ``via`` -- if that notebook is one of this scan's assets.
+
         The upstreams that are *not* part of this scan are the important half.
         They used to be dropped on the floor (``if target in table_hash_by_key``),
         which meant a table fed by a catalog outside the configured scope looked
@@ -686,6 +706,7 @@ class DatabricksSource(BaseTabularSource):
             return
 
         column_mappings = self._column_mappings_for_table(table_ref)
+        notebook_ids = self._lineage_notebook_ids.get(table_ref.table_key, {})
         for upstream in sorted(upstream_refs):
             upstream_hash = table_hash_by_key.get(upstream)
             upstream_ref = (
@@ -693,12 +714,21 @@ class DatabricksSource(BaseTabularSource):
             )
             if upstream_ref is None:
                 continue
+            via_hash = next(
+                (
+                    notebook_hash_by_id[notebook_id]
+                    for notebook_id in sorted(notebook_ids.get(upstream, ()))
+                    if notebook_id in (notebook_hash_by_id or {})
+                ),
+                None,
+            )
             self.add_edge(
                 flow(
                     upstream=upstream_ref,
                     downstream=Ref.asset(downstream_hash),
                     type=FlowType.TRANSFORM,
                     fields=column_mappings.get(upstream, []),
+                    via=Ref.asset(via_hash) if via_hash else None,
                     method=Method.SYSTEM_CATALOG,
                 )
             )
@@ -786,6 +816,7 @@ class DatabricksSource(BaseTabularSource):
                     "get", "/api/2.0/workspace/list", params={"path": path}
                 )
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning("Skipping workspace path %s due to listing error: %s", path, exc)
                 continue
             objects = payload.get("objects")
@@ -865,6 +896,7 @@ class DatabricksSource(BaseTabularSource):
             try:
                 payload = self._request_json("get", "/api/2.0/pipelines", params=params)
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning("Could not list Databricks pipelines: %s", exc)
                 break
             values: list[dict[str, Any]] = []
@@ -935,6 +967,15 @@ class DatabricksSource(BaseTabularSource):
             t.table_key: self.generate_hash_id(t.raw_id) for t in tables
         }
 
+        # Listed before the tables so a lineage edge can name the notebook that
+        # produced it; the notebook assets are still emitted after the tables.
+        notebooks = list(self._iter_notebooks())
+        notebook_hash_by_id = {
+            nb.object_id: self.generate_hash_id(f"notebook_#_{nb.path}")
+            for nb in notebooks
+            if nb.object_id
+        }
+
         include_lineage = self._extraction_options().include_table_lineage
         if include_lineage and tables:
             logger.info("Fetching table lineage for %d table(s)...", len(tables))
@@ -957,7 +998,9 @@ class DatabricksSource(BaseTabularSource):
                         for target in sorted(upstream_refs)
                         if target in table_hash_by_key
                     ]
-                    self._emit_table_lineage(table_ref, upstream_refs, table_hash_by_key)
+                    self._emit_table_lineage(
+                        table_ref, upstream_refs, table_hash_by_key, notebook_hash_by_id
+                    )
                 except Exception as exc:
                     logger.warning(
                         "Could not resolve Databricks lineage for %s: %s", table_label, exc
@@ -974,7 +1017,7 @@ class DatabricksSource(BaseTabularSource):
 
         # Notebooks
         notebook_count = 0
-        for notebook in self._iter_notebooks():
+        for notebook in notebooks:
             if self._aborted:
                 break
             batch.append(self._notebook_to_asset(notebook))
@@ -997,6 +1040,7 @@ class DatabricksSource(BaseTabularSource):
                 batch = []
         if pipeline_count:
             logger.info("Discovered %d pipeline(s)", pipeline_count)
+        self._declare_listing_if_whole()
 
         if batch:
             yield batch

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/src/sources/hive/source.py

@@ -293,7 +293,10 @@ class HiveSource(BaseTabularSource):
             upstreams = [
                 key
                 for key in upstream_tables_from_sql(
-                    definition, dialect="hive", default_database=view.database
+                    definition,
+                    dialect="hive",
+                    default_database=view.database,
+                    two_level=True,
                 )
                 if key != view.table_key and key in scoped
             ]

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/src/sources/mysql/source.py

@@ -322,7 +322,7 @@ class MySQLSource(BaseTabularSource):
                 upstreams = [
                     key
                     for key in upstream_tables_from_sql(
-                        definition, dialect="mysql", default_database=database
+                        definition, dialect="mysql", default_database=database, two_level=True
                     )
                     if key != view_key
                 ]

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/src/sources/neo4j/source.py

@@ -325,6 +325,9 @@ class Neo4jSource(BaseSource):
                 yield batch
                 batch = []
 
+        if not self._aborted and not self._scope_options().node_limit_per_label:
+            self.declare_listing_complete()
+
         if batch:
             logger.info("Emitting final batch of %d asset(s)", len(batch))
             yield batch

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/src/sources/tabular_base.py

@@ -32,7 +32,12 @@ from ..utils.file_metadata import build_columns
 from ..utils.file_parser import render_bytes_cell
 from ..utils.hashing import hash_id, unhash_id
 from ..utils.sampling_frontier import RangeFrontier
-from ..utils.sql_lineage import NO_UPSTREAM, column_mappings_by_table, sqlglot_dialect
+from ..utils.sql_lineage import (
+    NO_UPSTREAM,
+    column_mappings_by_table,
+    fields_for_table,
+    sqlglot_dialect,
+)
 from ..utils.urn import Urn, UrnError
 from .base import BaseSource
 from .tabular_utils import (
@@ -314,9 +319,23 @@ class BaseTabularSource(BaseSource):
             try:
                 tables.extend(self._list_tables_for_database(database))
             except Exception as exc:
+                self.note_listing_error()
                 logger.warning("Skipping database %s due to listing error: %s", database, exc)
         return tables
 
+    def _listing_is_limited(self) -> bool:
+        """Whether a configured cap can have cut the table list short."""
+        scope_options = getattr(self, "_scope_options", None)
+        scope = scope_options() if callable(scope_options) else None
+        return bool(
+            getattr(scope, "table_limit", None) or getattr(scope, "table_limit_per_schema", None)
+        )
+
+    def _declare_listing_if_whole(self) -> None:
+        """Offer this run's table list as the whole scope, when it plausibly is."""
+        if not self._aborted and not self._listing_is_limited():
+            self.declare_listing_complete()
+
     # ── Primary keys ─────────────────────────────────────────────────────
 
     def _get_primary_key_columns(self, table_ref: TableRef) -> list[str]:
@@ -1039,7 +1058,7 @@ class BaseTabularSource(BaseSource):
                     # spelled the name (a synonym, a quoted alias).
                     fields = [mapping for group in by_table.values() for mapping in group]
                 else:
-                    fields = list(by_table.get(str(upstream_key[-1]).lower(), []))
+                    fields = fields_for_table(by_table, upstream_key)
                 upstream_hash = table_hash_by_key.get(upstream_key)
                 upstream_ref = (
                     Ref.asset(upstream_hash)
@@ -1092,6 +1111,7 @@ class BaseTabularSource(BaseSource):
         table_hash_by_key: dict[tuple[str, ...], str] = {
             t.table_key: self.generate_hash_id(t.raw_id) for t in tables
         }
+        self._declare_listing_if_whole()
         fk_links = self._collect_foreign_key_links(tables)
         self._emit_relationship_edges(tables, table_hash_by_key, fk_links)
 

{classifyre_cli-0.6.18 → classifyre_cli-0.7.1}/src/utils/sql_lineage.py

@@ -32,6 +32,7 @@ logger = logging.getLogger(__name__)
 __all__ = [
     "NO_UPSTREAM",
     "column_mappings_by_table",
+    "fields_for_table",
     "sqlglot_dialect",
     "table_paths_from_sql",
     "upstream_tables_from_sql",
@@ -171,6 +172,114 @@ def _paths(node: Any, trail: tuple[Any, ...] = ()) -> Iterator[tuple[Any, ...]]:
         yield from _paths(child, trail)
 
 
+def _table_key(table: Any) -> str:
+    """A table as the SQL named it: ``schema.table``, or ``table`` when written bare.
+
+    The schema stays in the key because two tables can share a name across
+    schemas (``core.customers`` and ``edge.customers``) and a view can read both.
+    """
+    name = str(table.name).lower()
+    schema = str(table.db).lower()
+    return f"{schema}.{name}" if schema else name
+
+
+def fields_for_table(
+    by_table: dict[str, list[FieldMapping]], table_key: tuple[str, ...]
+) -> list[FieldMapping]:
+    """The mappings that belong to one upstream, given its catalog key.
+
+    ``table_key`` ends ``(…, schema, table)``. The schema-qualified spelling wins,
+    so same-named tables in two schemas are never confused; a table written bare
+    (it was on the search path) matches by name; and a name that only one
+    qualified entry carries is that entry, however the SQL spelled the schema.
+    """
+    name = str(table_key[-1]).lower()
+    if len(table_key) >= 2:
+        qualified = f"{str(table_key[-2]).lower()}.{name}"
+        if qualified in by_table:
+            return list(by_table[qualified])
+    if name in by_table:
+        return list(by_table[name])
+    candidates = [key for key in by_table if key.rsplit(".", 1)[-1] == name]
+    return list(by_table[candidates[0]]) if len(candidates) == 1 else []
+
+
+def _inline_lateral_subqueries(statement: Any, exp: Any) -> None:
+    """Make ``LATERAL (SELECT …) t`` an ordinary derived table.
+
+    sqlglot's lineage stops at a lateral and reports every column its subquery
+    mentions, including the outer table's correlated one, as feeding the output.
+    As a plain derived table the output columns follow their own projection to
+    the table they come from; the correlation is still found among the
+    indirect dependencies because scopes resolve names through their parents.
+    A lateral over a function (``jsonb_each(…)``) is left alone.
+    """
+    for lateral in list(statement.find_all(exp.Lateral)):
+        inner = lateral.this
+        if not isinstance(inner, exp.Subquery):
+            continue
+        alias = lateral.args.get("alias")
+        if alias is not None:
+            inner.set("alias", alias.copy())
+        lateral.replace(inner.copy())
+
+
+def _qualify_lone_sources(statement: Any, exp: Any) -> None:
+    """Name the table on columns written without one, where only one table is in play.
+
+    Postgres stores a single-table view with no qualifiers (``WHERE status <> …``),
+    and sqlglot will not place such a column without the table's schema. With one
+    table in the ``FROM`` there is only one place it can come from, so say so.
+    That also stops an output name that re-uses a column name (``qty::int AS qty``)
+    from being ambiguous the next time ``qty`` is read.
+
+    A name in ``ORDER BY`` / ``GROUP BY`` / ``HAVING`` that matches an output alias
+    is left for sqlglot to expand: it means the alias, not the column.
+    """
+    for select in list(statement.find_all(exp.Select)):
+        source = select.args.get("from_") or select.args.get("from")
+        if source is None or select.args.get("joins"):
+            continue
+        node = source.this
+        if isinstance(node, exp.Table) and isinstance(node.this, exp.Identifier):
+            qualifier = (node.args.get("alias") and node.args["alias"].this) or node.this
+        elif isinstance(node, exp.Subquery) and node.args.get("alias"):
+            qualifier = node.args["alias"].this
+        else:
+            continue
+        if not isinstance(qualifier, exp.Identifier):
+            continue
+
+        derived = {
+            projection.alias.lower()
+            for projection in select.expressions
+            if isinstance(projection, exp.Alias)
+            and not (
+                isinstance(projection.this, exp.Column)
+                and projection.this.name.lower() == projection.alias.lower()
+            )
+        }
+        for column in list(select.find_all(exp.Column)):
+            if column.table or not isinstance(column.this, exp.Identifier):
+                continue
+            if column.find_ancestor(exp.Select) is not select:
+                continue
+            tail = column.find_ancestor(exp.Order, exp.Group, exp.Having)
+            if tail is not None and tail.parent is select and column.name.lower() in derived:
+                continue
+            column.set("table", qualifier.copy())
+
+
+def _source_in_scope(scope: Any, name: str) -> Any:
+    """Look a table alias up in this scope, then in the scopes around it."""
+    while scope is not None:
+        found = scope.sources.get(name)
+        if found is not None:
+            return found
+        scope = scope.parent
+    return None
+
+
 def column_mappings_by_table(
     sql: str,
     *,
@@ -178,9 +287,10 @@ def column_mappings_by_table(
 ) -> dict[str, list[FieldMapping]]:
     """Per-column dependencies of a view, split by the table they come from.
 
-    Keys are lower-cased table names (the last part of the name), so a caller
-    holding ``(db, schema, table)`` upstream keys can attach to each edge only
-    the columns that table really feeds. :data:`NO_UPSTREAM` holds output
+    Keys are lower-cased table names as the SQL wrote them (``schema.table``, or
+    ``table`` when bare); :func:`fields_for_table` picks the entry for a catalog
+    key, so a caller can attach to each edge only the columns that table really
+    feeds. :data:`NO_UPSTREAM` holds output
     columns no table feeds.
 
     Columns are resolved through CTEs, derived tables and set operations, so a
@@ -214,6 +324,8 @@ def column_mappings_by_table(
             for projection in statement.copy().selects
             if projection.alias_or_name
         }
+        _inline_lateral_subqueries(statement, exp)
+        _qualify_lone_sources(statement, exp)
         statement = qualify(
             statement, dialect=dialect, validate_qualify_columns=False, identify=False
         )
@@ -234,7 +346,10 @@ def column_mappings_by_table(
             # which we deliberately do not resolve. Dataset-level it is.
             continue
         try:
-            node = lineage(name, statement, {}, dialect=dialect, scope=root)
+            # Quoted, so an alias like "Id" is matched as written instead of folded.
+            node = lineage(
+                exp.column(name, quoted=True), statement, {}, dialect=dialect, scope=root
+            )
         except Exception as exc:
             logger.debug("Could not trace column %s: %s", name, exc)
             continue
@@ -258,8 +373,8 @@ def column_mappings_by_table(
                     constant = _Trace()
                     constant.add(projections, None, exp)
                 continue
-            table = str(leaf.source.name).lower()
-            column = leaf.name.rsplit(".", 1)[-1]
+            table = _table_key(leaf.source)
+            column = leaf.name.rsplit(".", 1)[-1].strip('"`')
             if table and column and column != "*":
                 per_table.setdefault(table, _Trace()).add(projections, column, exp)
 
@@ -312,9 +427,9 @@ def column_mappings_by_table(
             if clause is None:
                 continue
             for column in clause.find_all(exp.Column):
-                source = scope.sources.get(column.table)
+                source = _source_in_scope(scope, column.table)
                 if isinstance(source, exp.Table) and column.name and column.name != "*":
-                    indirect.setdefault(str(source.name).lower(), set()).add(column.name)
+                    indirect.setdefault(_table_key(source), set()).add(column.name)
 
     for table, columns in indirect.items():
         by_table.setdefault(table, []).append(
@@ -329,6 +444,7 @@ def upstream_tables_from_sql(
     dialect: str | None = None,
     default_database: str | None = None,
     default_schema: str | None = None,
+    two_level: bool = False,
 ) -> list[tuple[str, ...]]:
     """Table keys a statement reads from, for catalogs with no dependency view.
 
@@ -338,6 +454,11 @@ def upstream_tables_from_sql(
     the wrong default schema points at nothing — so callers should prefer the
     catalog wherever it exists and treat these edges as SQL_PARSED.
 
+    ``two_level`` is for engines whose tables are named ``database.table`` (MySQL,
+    Hive). sqlglot files the first part under ``db``, so without it a qualified
+    name came back as ``(default_database, database, table)`` -- a key no asset
+    has, which turned every such view's edges into unresolved ones.
+
     CTE names are excluded: a ``WITH`` alias is not a table, and emitting it
     would create lineage to an object that does not exist.
     """
@@ -366,6 +487,11 @@ def upstream_tables_from_sql(
         name = table.name
         if not name or name.lower() in cte_names:
             continue
+        if two_level:
+            namespace = table.text("db") or default_database
+            if namespace:
+                keys.add((namespace, name))
+            continue
         schema = table.text("db") or default_schema
         database = table.text("catalog") or default_database
         if database and schema: