classifyre-cli 0.6.18__tar.gz → 0.7.0__tar.gz

{classifyre_cli-0.6.18 → classifyre_cli-0.7.0}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 325 packages in 164ms
+Resolved 325 packages in 165ms
 Checked 52 packages in 2ms

{classifyre_cli-0.6.18 → classifyre_cli-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.6.18
+Version: 0.7.0
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.6.18 → classifyre_cli-0.7.0}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.6.18",
+  "version": "0.7.0",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.6.18 → classifyre_cli-0.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.6.18"
+version = "0.7.0"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably

{classifyre_cli-0.6.18 → classifyre_cli-0.7.0}/src/sources/tabular_base.py

@@ -32,7 +32,12 @@ from ..utils.file_metadata import build_columns
 from ..utils.file_parser import render_bytes_cell
 from ..utils.hashing import hash_id, unhash_id
 from ..utils.sampling_frontier import RangeFrontier
-from ..utils.sql_lineage import NO_UPSTREAM, column_mappings_by_table, sqlglot_dialect
+from ..utils.sql_lineage import (
+    NO_UPSTREAM,
+    column_mappings_by_table,
+    fields_for_table,
+    sqlglot_dialect,
+)
 from ..utils.urn import Urn, UrnError
 from .base import BaseSource
 from .tabular_utils import (
@@ -1039,7 +1044,7 @@ class BaseTabularSource(BaseSource):
                     # spelled the name (a synonym, a quoted alias).
                     fields = [mapping for group in by_table.values() for mapping in group]
                 else:
-                    fields = list(by_table.get(str(upstream_key[-1]).lower(), []))
+                    fields = fields_for_table(by_table, upstream_key)
                 upstream_hash = table_hash_by_key.get(upstream_key)
                 upstream_ref = (
                     Ref.asset(upstream_hash)

{classifyre_cli-0.6.18 → classifyre_cli-0.7.0}/src/utils/sql_lineage.py

@@ -32,6 +32,7 @@ logger = logging.getLogger(__name__)
 __all__ = [
     "NO_UPSTREAM",
     "column_mappings_by_table",
+    "fields_for_table",
     "sqlglot_dialect",
     "table_paths_from_sql",
     "upstream_tables_from_sql",
@@ -171,6 +172,114 @@ def _paths(node: Any, trail: tuple[Any, ...] = ()) -> Iterator[tuple[Any, ...]]:
         yield from _paths(child, trail)
 
 
+def _table_key(table: Any) -> str:
+    """A table as the SQL named it: ``schema.table``, or ``table`` when written bare.
+
+    The schema stays in the key because two tables can share a name across
+    schemas (``core.customers`` and ``edge.customers``) and a view can read both.
+    """
+    name = str(table.name).lower()
+    schema = str(table.db).lower()
+    return f"{schema}.{name}" if schema else name
+
+
+def fields_for_table(
+    by_table: dict[str, list[FieldMapping]], table_key: tuple[str, ...]
+) -> list[FieldMapping]:
+    """The mappings that belong to one upstream, given its catalog key.
+
+    ``table_key`` ends ``(…, schema, table)``. The schema-qualified spelling wins,
+    so same-named tables in two schemas are never confused; a table written bare
+    (it was on the search path) matches by name; and a name that only one
+    qualified entry carries is that entry, however the SQL spelled the schema.
+    """
+    name = str(table_key[-1]).lower()
+    if len(table_key) >= 2:
+        qualified = f"{str(table_key[-2]).lower()}.{name}"
+        if qualified in by_table:
+            return list(by_table[qualified])
+    if name in by_table:
+        return list(by_table[name])
+    candidates = [key for key in by_table if key.rsplit(".", 1)[-1] == name]
+    return list(by_table[candidates[0]]) if len(candidates) == 1 else []
+
+
+def _inline_lateral_subqueries(statement: Any, exp: Any) -> None:
+    """Make ``LATERAL (SELECT …) t`` an ordinary derived table.
+
+    sqlglot's lineage stops at a lateral and reports every column its subquery
+    mentions, including the outer table's correlated one, as feeding the output.
+    As a plain derived table the output columns follow their own projection to
+    the table they come from; the correlation is still found among the
+    indirect dependencies because scopes resolve names through their parents.
+    A lateral over a function (``jsonb_each(…)``) is left alone.
+    """
+    for lateral in list(statement.find_all(exp.Lateral)):
+        inner = lateral.this
+        if not isinstance(inner, exp.Subquery):
+            continue
+        alias = lateral.args.get("alias")
+        if alias is not None:
+            inner.set("alias", alias.copy())
+        lateral.replace(inner.copy())
+
+
+def _qualify_lone_sources(statement: Any, exp: Any) -> None:
+    """Name the table on columns written without one, where only one table is in play.
+
+    Postgres stores a single-table view with no qualifiers (``WHERE status <> …``),
+    and sqlglot will not place such a column without the table's schema. With one
+    table in the ``FROM`` there is only one place it can come from, so say so.
+    That also stops an output name that re-uses a column name (``qty::int AS qty``)
+    from being ambiguous the next time ``qty`` is read.
+
+    A name in ``ORDER BY`` / ``GROUP BY`` / ``HAVING`` that matches an output alias
+    is left for sqlglot to expand: it means the alias, not the column.
+    """
+    for select in list(statement.find_all(exp.Select)):
+        source = select.args.get("from_") or select.args.get("from")
+        if source is None or select.args.get("joins"):
+            continue
+        node = source.this
+        if isinstance(node, exp.Table) and isinstance(node.this, exp.Identifier):
+            qualifier = (node.args.get("alias") and node.args["alias"].this) or node.this
+        elif isinstance(node, exp.Subquery) and node.args.get("alias"):
+            qualifier = node.args["alias"].this
+        else:
+            continue
+        if not isinstance(qualifier, exp.Identifier):
+            continue
+
+        derived = {
+            projection.alias.lower()
+            for projection in select.expressions
+            if isinstance(projection, exp.Alias)
+            and not (
+                isinstance(projection.this, exp.Column)
+                and projection.this.name.lower() == projection.alias.lower()
+            )
+        }
+        for column in list(select.find_all(exp.Column)):
+            if column.table or not isinstance(column.this, exp.Identifier):
+                continue
+            if column.find_ancestor(exp.Select) is not select:
+                continue
+            tail = column.find_ancestor(exp.Order, exp.Group, exp.Having)
+            if tail is not None and tail.parent is select and column.name.lower() in derived:
+                continue
+            column.set("table", qualifier.copy())
+
+
+def _source_in_scope(scope: Any, name: str) -> Any:
+    """Look a table alias up in this scope, then in the scopes around it."""
+    while scope is not None:
+        found = scope.sources.get(name)
+        if found is not None:
+            return found
+        scope = scope.parent
+    return None
+
+
 def column_mappings_by_table(
     sql: str,
     *,
@@ -178,9 +287,10 @@ def column_mappings_by_table(
 ) -> dict[str, list[FieldMapping]]:
     """Per-column dependencies of a view, split by the table they come from.
 
-    Keys are lower-cased table names (the last part of the name), so a caller
-    holding ``(db, schema, table)`` upstream keys can attach to each edge only
-    the columns that table really feeds. :data:`NO_UPSTREAM` holds output
+    Keys are lower-cased table names as the SQL wrote them (``schema.table``, or
+    ``table`` when bare); :func:`fields_for_table` picks the entry for a catalog
+    key, so a caller can attach to each edge only the columns that table really
+    feeds. :data:`NO_UPSTREAM` holds output
     columns no table feeds.
 
     Columns are resolved through CTEs, derived tables and set operations, so a
@@ -214,6 +324,8 @@ def column_mappings_by_table(
             for projection in statement.copy().selects
             if projection.alias_or_name
         }
+        _inline_lateral_subqueries(statement, exp)
+        _qualify_lone_sources(statement, exp)
         statement = qualify(
             statement, dialect=dialect, validate_qualify_columns=False, identify=False
         )
@@ -234,7 +346,10 @@ def column_mappings_by_table(
             # which we deliberately do not resolve. Dataset-level it is.
             continue
         try:
-            node = lineage(name, statement, {}, dialect=dialect, scope=root)
+            # Quoted, so an alias like "Id" is matched as written instead of folded.
+            node = lineage(
+                exp.column(name, quoted=True), statement, {}, dialect=dialect, scope=root
+            )
         except Exception as exc:
             logger.debug("Could not trace column %s: %s", name, exc)
             continue
@@ -258,8 +373,8 @@ def column_mappings_by_table(
                     constant = _Trace()
                     constant.add(projections, None, exp)
                 continue
-            table = str(leaf.source.name).lower()
-            column = leaf.name.rsplit(".", 1)[-1]
+            table = _table_key(leaf.source)
+            column = leaf.name.rsplit(".", 1)[-1].strip('"`')
             if table and column and column != "*":
                 per_table.setdefault(table, _Trace()).add(projections, column, exp)
 
@@ -312,9 +427,9 @@ def column_mappings_by_table(
             if clause is None:
                 continue
             for column in clause.find_all(exp.Column):
-                source = scope.sources.get(column.table)
+                source = _source_in_scope(scope, column.table)
                 if isinstance(source, exp.Table) and column.name and column.name != "*":
-                    indirect.setdefault(str(source.name).lower(), set()).add(column.name)
+                    indirect.setdefault(_table_key(source), set()).add(column.name)
 
     for table, columns in indirect.items():
         by_table.setdefault(table, []).append(

{classifyre_cli-0.6.18 → classifyre_cli-0.7.0}/tests/test_sql_lineage.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from src.graph.edges import FieldMapping, FieldTransform
-from src.utils.sql_lineage import NO_UPSTREAM, column_mappings_by_table
+from src.utils.sql_lineage import NO_UPSTREAM, column_mappings_by_table, fields_for_table
 
 
 def _by_downstream(mappings: list[FieldMapping]) -> dict[str | None, FieldMapping]:
@@ -102,7 +102,7 @@ def test_upper_case_snowflake_names_resolve() -> None:
         "SELECT ID, TRIM(NAME) AS NAME FROM ANALYTICS.PUBLIC.ACCOUNTS", dialect="snowflake"
     )
 
-    name = _by_downstream(by_table["accounts"])["NAME"]
+    name = _by_downstream(by_table["public.accounts"])["NAME"]
     assert name.upstreams == ("NAME",)
     assert name.steps == ("TRIM",)
 
@@ -134,8 +134,8 @@ def test_postgres_stored_view_text() -> None:
         dialect="postgres",
     )
 
-    assert _by_downstream(by_table["orders"])[None].upstreams == ("customer_id", "status")
-    assert _by_downstream(by_table["customers"])[None].upstreams == ("id",)
+    assert _by_downstream(by_table["shop.orders"])[None].upstreams == ("customer_id", "status")
+    assert _by_downstream(by_table["shop.customers"])[None].upstreams == ("id",)
     assert _by_downstream(by_table[NO_UPSTREAM])["channel"].steps == ()
 
 
@@ -146,8 +146,133 @@ def test_each_union_branch_keeps_its_own_kind_and_expression() -> None:
         dialect="postgres",
     )
 
-    orders = _by_downstream(by_table["orders"])["amount"]
-    refunds = _by_downstream(by_table["refunds"])["amount"]
+    orders = _by_downstream(by_table["shop.orders"])["amount"]
+    refunds = _by_downstream(by_table["shop.refunds"])["amount"]
     assert orders.type is FieldTransform.IDENTITY
     assert refunds.steps == ("-",)
     assert refunds.transform == "(-refunds.amount)"
+
+
+# ── Postgres stored text: what pg_get_viewdef really writes ──────────────
+#
+# A single-table view is stored with no qualifiers on its columns at all, which
+# is the shape sqlglot cannot place without a schema.
+
+
+def test_single_table_view_keeps_where_group_and_order_columns_as_indirect() -> None:
+    by_table = column_mappings_by_table(
+        """ SELECT order_date,
+    sum(gross_amount) AS revenue
+   FROM lab_mart.fct_orders f
+  WHERE status <> 'CANCELLED'::text
+  GROUP BY order_date
+  ORDER BY order_date;""",
+        dialect="postgres",
+    )
+
+    mappings = _by_downstream(by_table["lab_mart.fct_orders"])
+    assert mappings["revenue"].upstreams == ("gross_amount",)
+    assert mappings[None].type is FieldTransform.INDIRECT
+    assert mappings[None].upstreams == ("order_date", "status")
+
+
+def test_output_name_that_shadows_a_source_column_can_be_used_again() -> None:
+    # `qty::integer AS qty` renames in place, then `qty` is used a second time.
+    by_table = column_mappings_by_table(
+        """ SELECT order_ref,
+    qty::integer AS qty,
+    qty::integer::numeric * unit_price::numeric AS line_total
+   FROM lab_raw.order_lines_raw l;""",
+        dialect="postgres",
+    )
+
+    mappings = _by_downstream(by_table["lab_raw.order_lines_raw"])
+    assert mappings["qty"].upstreams == ("qty",)
+    assert mappings["line_total"].upstreams == ("qty", "unit_price")
+
+
+def test_order_by_an_output_alias_is_not_mistaken_for_a_column() -> None:
+    by_table = column_mappings_by_table(
+        "SELECT region, SUM(amount) AS total FROM sales GROUP BY region ORDER BY total DESC",
+        dialect="postgres",
+    )
+
+    mappings = _by_downstream(by_table["sales"])
+    assert mappings["total"].upstreams == ("amount",)
+    assert mappings[None].upstreams == ("region",)
+
+
+def test_quoted_mixed_case_output_names_resolve() -> None:
+    by_table = column_mappings_by_table(
+        """ SELECT "Order ID" AS "Id",
+    lower("Customer Email") AS "Mail",
+    "Total (EUR)" * 1.19 AS "Gross"
+   FROM lab_edge."Mixed Case Orders" m;""",
+        dialect="postgres",
+    )
+
+    mappings = _by_downstream(by_table["lab_edge.mixed case orders"])
+    assert mappings["Id"].upstreams == ("Order ID",)
+    assert mappings["Mail"].steps == ("LOWER",)
+    assert mappings["Gross"].upstreams == ("Total (EUR)",)
+
+
+def test_same_table_name_in_two_schemas_is_kept_apart() -> None:
+    by_table = column_mappings_by_table(
+        """ SELECT c.email,
+    e.loyalty_tier
+   FROM lab_core.customers c
+     JOIN lab_edge.customers e ON e.customer_id = c.customer_id;""",
+        dialect="postgres",
+    )
+
+    core = _by_downstream(by_table["lab_core.customers"])
+    edge = _by_downstream(by_table["lab_edge.customers"])
+    assert set(core) == {"email", None}
+    assert set(edge) == {"loyalty_tier", None}
+
+
+def test_lateral_subquery_is_followed_to_its_own_table() -> None:
+    by_table = column_mappings_by_table(
+        """ SELECT c.customer_id,
+    t.order_id,
+    t.total_amount
+   FROM lab_core.customers c
+     CROSS JOIN LATERAL ( SELECT o.order_id,
+            o.total_amount
+           FROM lab_core.orders o
+          WHERE o.customer_id = c.customer_id
+          ORDER BY o.total_amount DESC
+         LIMIT 3) t;""",
+        dialect="postgres",
+    )
+
+    customers = _by_downstream(by_table["lab_core.customers"])
+    orders = _by_downstream(by_table["lab_core.orders"])
+    assert set(customers) == {"customer_id", None}
+    assert orders["order_id"].upstreams == ("order_id",)
+    assert orders["total_amount"].upstreams == ("total_amount",)
+    # The correlation `o.customer_id = c.customer_id` shapes rows from both.
+    assert "customer_id" in orders[None].upstreams
+    assert customers[None].upstreams == ("customer_id",)
+
+
+def test_fields_for_table_prefers_the_schema_it_was_written_with() -> None:
+    by_table = column_mappings_by_table(
+        "SELECT c.email, e.tier FROM lab_core.customers c "
+        "JOIN lab_edge.customers e ON e.customer_id = c.customer_id",
+        dialect="postgres",
+    )
+
+    core = fields_for_table(by_table, ("db", "lab_core", "customers"))
+    edge = fields_for_table(by_table, ("db", "lab_edge", "customers"))
+    assert {m.downstream for m in core} == {"email", None}
+    assert {m.downstream for m in edge} == {"tier", None}
+
+
+def test_fields_for_table_matches_a_bare_name_by_table() -> None:
+    # On the search path Postgres writes the table without its schema.
+    by_table = column_mappings_by_table("SELECT id FROM orders", dialect="postgres")
+
+    assert [m.downstream for m in fields_for_table(by_table, ("db", "public", "orders"))] == ["id"]
+    assert fields_for_table(by_table, ("db", "public", "refunds")) == []