coltrace 0.1.0.dev7__py3-none-any.whl → 0.1.0.dev8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
coltrace/__init__.py CHANGED
@@ -7,7 +7,7 @@ from .models import ScanReport
7
7
  from .scanner import ScanConfig, scan_project
8
8
  from .source_models import SourceContract, SourceSchema, schema_from_columns
9
9
 
10
- __version__ = "0.1.0.dev7"
10
+ __version__ = "0.1.0.dev8"
11
11
  __all__ = [
12
12
  "LineageReport",
13
13
  "ScanConfig",
coltrace/analysis.py CHANGED
@@ -12,6 +12,7 @@ import fnmatch
12
12
  import hashlib
13
13
  import io
14
14
  import operator
15
+ import re
15
16
  import sys
16
17
  import tokenize
17
18
  import warnings
@@ -136,6 +137,10 @@ class _Value:
136
137
 
137
138
 
138
139
  PLACEHOLDER = "__coltrace_param_{}__"
140
+ PLACEHOLDER_PREFIX = "__coltrace_param_"
141
+ _PLACEHOLDER_NAME = re.compile(r"__coltrace_param_\d+__")
142
+ # Method names whose receiver dialect is a useful SQL parsing hint.
143
+ _DIALECT_HINTS = {"sql": "databricks", "query": "bigquery", "query_and_wait": "bigquery"}
139
144
 
140
145
  # Library constants: scalar literals, not objects that could hold data.
141
146
  CONSTANT_SYMBOLS = frozenset(
@@ -219,6 +224,8 @@ class _Opaque:
219
224
  conversion: bool = False
220
225
  # Labels declared for the resulting DataFrame, such as columns=[...].
221
226
  columns: tuple[str, ...] | None = None
227
+ # SQL dialect suggested by the call shape; parsing still falls back.
228
+ dialect: str | None = None
222
229
 
223
230
 
224
231
  @dataclass
@@ -382,8 +389,16 @@ class _Analyzer:
382
389
  source_contracts: tuple[SourceContract, ...] = (),
383
390
  source_schemas: tuple[SourceSchema, ...] = (),
384
391
  observers: tuple[str, ...] = (),
392
+ sql_dialect: str | None = None,
385
393
  ) -> None:
386
394
  self.observers = observers
395
+ self.sql_dialect = sql_dialect
396
+ # Unknown interpolations in SQL text: stable names per source position.
397
+ self.placeholders: dict[str, str] = {}
398
+ self.placeholder_sites: dict[tuple[str, int, int, int, int], str] = {}
399
+ # Database relations read by SQL sources, and their physical columns.
400
+ self.table_frames: dict[str, str] = {}
401
+ self.table_columns: dict[tuple[str, str], str] = {}
387
402
  self.modules = modules
388
403
  self.ambiguous = ambiguous
389
404
  self.frames: list[Frame] = []
@@ -693,6 +708,7 @@ class _Analyzer:
693
708
  context: _Context,
694
709
  node: ast.AST,
695
710
  kind: str = "derive",
711
+ operation_kind: str | None = None,
696
712
  ) -> str:
697
713
  value = self._refresh(value)
698
714
  state.revision += 1
@@ -746,7 +762,9 @@ class _Analyzer:
746
762
  state.removed.discard(name)
747
763
  if state.schema is not None:
748
764
  state.schema.add(name)
749
- operation = self._operation(context, node, kind, column.id, evidence, value.completeness)
765
+ operation = self._operation(
766
+ context, node, operation_kind or kind, column.id, evidence, value.completeness
767
+ )
750
768
  for dependency in dict.fromkeys(value.dependencies):
751
769
  self.edges.append(Edge(dependency, operation.id, "read", evidence))
752
770
  self.edges.append(Edge(operation.id, column.id, "write", evidence))
@@ -1439,7 +1457,14 @@ class _Analyzer:
1439
1457
  identifiers.extend(self.columns[column].dataframe_id for column in value.dependencies)
1440
1458
  return tuple(dict.fromkeys(identifiers))
1441
1459
 
1442
- def _consume(self, value: _Value, context: _Context, node: ast.AST, kind: str) -> str | None:
1460
+ def _consume(
1461
+ self,
1462
+ value: _Value,
1463
+ context: _Context,
1464
+ node: ast.AST,
1465
+ kind: str,
1466
+ label: str | None = None,
1467
+ ) -> str | None:
1443
1468
  dependencies = value.dependencies
1444
1469
  if value.frame is not None:
1445
1470
  dependencies = tuple(value.frame.current.values())
@@ -1467,7 +1492,7 @@ class _Analyzer:
1467
1492
  context,
1468
1493
  node,
1469
1494
  event_kind,
1470
- kind.replace("_", " "),
1495
+ label or kind.replace("_", " "),
1471
1496
  frames=frames,
1472
1497
  inputs=dependencies,
1473
1498
  parents=self._parents(value)
@@ -1686,18 +1711,18 @@ class _Analyzer:
1686
1711
  return folded
1687
1712
  if (
1688
1713
  isinstance(node.op, ast.Add)
1689
- and all(value.kind == "literal" for value in values)
1714
+ and all(value.kind == "literal" or self._plain_unknown(value) for value in values)
1690
1715
  and (self._text(left) is not None or self._text(right) is not None)
1691
1716
  ):
1692
1717
  # A partially known string keeps its known text for SQL parsing.
1693
1718
  return _Value(
1694
1719
  "literal",
1695
1720
  completeness="complete",
1696
- template=(self._text(left) or PLACEHOLDER.format("l"))
1697
- + (self._text(right) or PLACEHOLDER.format("r")),
1721
+ template=(self._text(left) or self._placeholder(node.left, context))
1722
+ + (self._text(right) or self._placeholder(node.right, context)),
1698
1723
  )
1699
1724
  if isinstance(node.op, ast.Mod) and left.known and isinstance(left.constant, str):
1700
- return _Value("literal", completeness="complete")
1725
+ return self._percent_format(node, left.constant, right, context)
1701
1726
  if isinstance(node.op, ast.Add):
1702
1727
  first, second = self._elements(left), self._elements(right)
1703
1728
  if (
@@ -1721,6 +1746,65 @@ class _Analyzer:
1721
1746
  )
1722
1747
  return self._degraded(values, kind=kind)
1723
1748
 
1749
+ def _placeholder(self, node: ast.AST, context: _Context) -> str:
1750
+ """A stable SQL-safe name for one unknown interpolation, labeled by its code."""
1751
+ key = (
1752
+ context.module.name,
1753
+ getattr(node, "lineno", 0),
1754
+ getattr(node, "col_offset", 0),
1755
+ getattr(node, "end_lineno", 0),
1756
+ getattr(node, "end_col_offset", 0),
1757
+ )
1758
+ name = self.placeholder_sites.get(key)
1759
+ if name is None:
1760
+ name = PLACEHOLDER.format(len(self.placeholders) + 1)
1761
+ label = " ".join(self._code(context, node).split())
1762
+ if len(label) > 60:
1763
+ label = label[:57] + "..."
1764
+ self.placeholders[name] = "{" + label + "}"
1765
+ self.placeholder_sites[key] = name
1766
+ return name
1767
+
1768
+ def _display_sql(self, text: str) -> str:
1769
+ """Show unknown interpolations as their Python expression, e.g. ``{table}``."""
1770
+ return _PLACEHOLDER_NAME.sub(
1771
+ lambda match: self.placeholders.get(match.group(0), "{?}"), text
1772
+ )
1773
+
1774
+ @staticmethod
1775
+ def _plain_unknown(value: _Value) -> bool:
1776
+ """An unknown scalar-like value: not a frame, column, object or collection."""
1777
+ return (
1778
+ value.kind == "unknown"
1779
+ and value.frame is None
1780
+ and not value.dependencies
1781
+ and value.instance is None
1782
+ and value.function is None
1783
+ and value.items is None
1784
+ and value.members is None
1785
+ and value.opaque is None
1786
+ )
1787
+
1788
+ def _percent_format(
1789
+ self, node: ast.BinOp, text: str, right: _Value, context: _Context
1790
+ ) -> _Value:
1791
+ """``"..." % values``: fold known values, otherwise keep a SQL template."""
1792
+ if right.known and isinstance(right.constant, (tuple, dict, str, int, float, bool)):
1793
+ try:
1794
+ result = text % right.constant
1795
+ except (TypeError, ValueError, KeyError, OverflowError):
1796
+ return _Value("literal", completeness="complete")
1797
+ if len(result) <= 1_000_000:
1798
+ return self._literal_value(result)
1799
+ return _Value("literal", completeness="complete")
1800
+ name = self._placeholder(node.right, context)
1801
+ template = re.sub(
1802
+ r"%%|%(\([^)]*\))?[-#0 +]*\d*(\.\d+)?[sdifrxXeEgGc]",
1803
+ lambda match: "%" if match.group(0) == "%%" else name,
1804
+ text,
1805
+ )
1806
+ return _Value("literal", completeness="complete", template=template)
1807
+
1724
1808
  def _text(self, value: _Value) -> str | None:
1725
1809
  """String text, with placeholders for unknown interpolated parts."""
1726
1810
  if value.known and isinstance(value.constant, str):
@@ -1868,7 +1952,7 @@ class _Analyzer:
1868
1952
  pattern.append(
1869
1953
  item.template
1870
1954
  if item.template is not None and formatted.format_spec is None
1871
- else PLACEHOLDER.format(len(pattern))
1955
+ else self._placeholder(formatted.value, context)
1872
1956
  )
1873
1957
  if formatted.format_spec is not None:
1874
1958
  values.append(self._expr(formatted.format_spec, context))
@@ -1981,7 +2065,9 @@ class _Analyzer:
1981
2065
  node: ast.ListComp | ast.SetComp | ast.GeneratorExp | ast.DictComp,
1982
2066
  context: _Context,
1983
2067
  ) -> _Value | None:
1984
- if len(node.generators) != 1 or isinstance(node, ast.GeneratorExp):
2068
+ # A generator over known constants evaluates like a list: its consumer
2069
+ # (join, tuple, any, ...) then folds statically.
2070
+ if len(node.generators) != 1:
1985
2071
  return None
1986
2072
  generator = node.generators[0]
1987
2073
  if generator.is_async or not isinstance(generator.target, (ast.Name, ast.Tuple)):
@@ -1999,6 +2085,10 @@ class _Analyzer:
1999
2085
  for call in calls
2000
2086
  ):
2001
2087
  return None
2088
+ if isinstance(node, ast.GeneratorExp) and calls:
2089
+ # Only pure element expressions (names, constants, f-strings) are
2090
+ # evaluated per item; anything else keeps the one-element summary.
2091
+ return None
2002
2092
  iterable = self._expr(generator.iter, context)
2003
2093
  if (
2004
2094
  not iterable.known
@@ -2146,7 +2236,7 @@ class _Analyzer:
2146
2236
  and node.func.attr == "format"
2147
2237
  and receiver.template is not None
2148
2238
  ):
2149
- formatted = self._format(receiver.template, arguments, keywords)
2239
+ formatted = self._format(receiver.template, arguments, keywords, node, context)
2150
2240
  if formatted is not None:
2151
2241
  text = (
2152
2242
  formatted.template if formatted.template is not None else formatted.constant
@@ -2319,7 +2409,7 @@ class _Analyzer:
2319
2409
  "Call effects and return provenance are unresolved; affected state is invalidated.",
2320
2410
  )
2321
2411
  self._invalidate(values, context, node)
2322
- return self._opaque_result(node, context, function, arguments, keywords, values)
2412
+ return self._opaque_result(node, context, function, arguments, keywords, values, receiver)
2323
2413
 
2324
2414
  def _column_label(
2325
2415
  self, node: ast.Subscript, context: _Context
@@ -2837,7 +2927,7 @@ class _Analyzer:
2837
2927
  allowed = CONSTANT_METHODS.get(type(receiver.constant), frozenset())
2838
2928
  if method not in allowed:
2839
2929
  if method == "format" and isinstance(receiver.constant, str):
2840
- return self._format(receiver.constant, arguments, keywords)
2930
+ return self._format(receiver.constant, arguments, keywords, node, context)
2841
2931
  return None
2842
2932
  try:
2843
2933
  result = getattr(receiver.constant, method)(*args, **kwargs)
@@ -2849,9 +2939,13 @@ class _Analyzer:
2849
2939
  return None
2850
2940
  return self._literal_value(result)
2851
2941
 
2852
- @staticmethod
2853
2942
  def _format(
2854
- text: str, arguments: list[_Value], keywords: list[tuple[str | None, _Value]]
2943
+ self,
2944
+ text: str,
2945
+ arguments: list[_Value],
2946
+ keywords: list[tuple[str | None, _Value]],
2947
+ node: ast.Call,
2948
+ context: _Context,
2855
2949
  ) -> _Value | None:
2856
2950
  """``str.format`` with simple field names; unknown values become placeholders."""
2857
2951
  import string
@@ -2864,10 +2958,17 @@ class _Analyzer:
2864
2958
  return None
2865
2959
  known = all(item.known for item in arguments) and all(item.known for _, item in keywords)
2866
2960
 
2961
+ sources: dict[object, ast.AST] = dict(enumerate(node.args))
2962
+ sources.update(
2963
+ (keyword.arg, keyword.value) for keyword in node.keywords if keyword.arg is not None
2964
+ )
2965
+
2867
2966
  def shown(item: _Value, index: object) -> object:
2868
2967
  if item.known and isinstance(item.constant, (str, int, float, bool)):
2869
2968
  return item.constant
2870
- return item.template or PLACEHOLDER.format(index)
2969
+ if item.template:
2970
+ return item.template
2971
+ return self._placeholder(sources.get(index, node), context)
2871
2972
 
2872
2973
  try:
2873
2974
  result = text.format(
@@ -2877,7 +2978,7 @@ class _Analyzer:
2877
2978
  except (IndexError, KeyError, ValueError):
2878
2979
  return None
2879
2980
  if known:
2880
- return _Analyzer._literal_value(result)
2981
+ return self._literal_value(result)
2881
2982
  return _Value("literal", completeness="complete", template=result)
2882
2983
 
2883
2984
  def _derived_call(
@@ -4361,34 +4462,71 @@ class _Analyzer:
4361
4462
  relations: tuple[str, ...] = (item.relation,) if item.relation else ()
4362
4463
  controls: tuple[str, ...] = ()
4363
4464
  complete = False
4465
+ # Physical (relation, column) inputs per output field, and SQL controls.
4466
+ references: dict[str, tuple[tuple[str, str], ...]] = {}
4467
+ control_references: tuple[tuple[str, str], ...] = ()
4364
4468
  if item.kind == "sql":
4365
4469
  if item.query is not None:
4366
4470
  from .sql_sources import summarize_sql
4367
4471
 
4368
4472
  try:
4369
- summary = summarize_sql(item.query, item.dialect)
4473
+ summary = summarize_sql(
4474
+ item.query,
4475
+ item.dialect or self.sql_dialect,
4476
+ fallback=True,
4477
+ placeholder=PLACEHOLDER_PREFIX,
4478
+ )
4370
4479
  except ValueError as error:
4371
4480
  self._diagnose(context, node, "CT_UNRESOLVED_SQL", str(error))
4372
4481
  else:
4373
- marker = PLACEHOLDER.split("{")[0]
4482
+ marker = PLACEHOLDER_PREFIX
4483
+ static = [field for field in summary.fields if marker not in field.name]
4374
4484
  fields = tuple(
4375
- SourceField(field.name, field.expression, field.inputs)
4376
- for field in summary.fields
4377
- if marker not in field.name
4485
+ SourceField(
4486
+ field.name,
4487
+ self._display_sql(field.expression),
4488
+ tuple(self._display_sql(name) for name in field.inputs),
4489
+ )
4490
+ for field in static
4378
4491
  )
4379
- relations, controls, complete = (
4380
- tuple(item for item in summary.relations if marker not in item),
4381
- summary.controls,
4382
- summary.complete and len(fields) == len(summary.fields),
4492
+ references = {field.name: field.references for field in static}
4493
+ control_references = tuple(
4494
+ reference
4495
+ for reference in summary.control_references
4496
+ if marker not in reference[1]
4383
4497
  )
4384
- if marker in item.query:
4498
+ relations = tuple(self._display_sql(name) for name in summary.relations)
4499
+ controls = tuple(
4500
+ self._display_sql(name)
4501
+ for name in summary.controls
4502
+ if marker not in name.rsplit(".", 1)[-1]
4503
+ )
4504
+ complete = summary.complete and len(static) == len(summary.fields)
4505
+ interpolations = list(dict.fromkeys(_PLACEHOLDER_NAME.findall(item.query)))
4506
+ if interpolations:
4507
+ shown = ", ".join(
4508
+ self.placeholders.get(name, "{?}") for name in interpolations[:4]
4509
+ )
4510
+ more = len(interpolations) - 4
4385
4511
  self._note(
4512
+ context,
4513
+ node,
4514
+ "CT_SQL_INTERPOLATION",
4515
+ f"SQL text has {len(interpolations)} interpolation(s) that are not "
4516
+ f"statically known ({shown}{f', +{more}' if more > 0 else ''}). "
4517
+ "The known text is parsed; relations and columns named by them "
4518
+ "are shown as {expression} and fields reading them are partial.",
4519
+ )
4520
+ if len(static) != len(summary.fields):
4521
+ self._diagnose(
4386
4522
  context,
4387
4523
  node,
4388
4524
  "CT_PARTIAL_SQL",
4389
- "SQL contains interpolations that are not statically known; they "
4390
- "were replaced by placeholders and are not modeled as fields.",
4525
+ "Some SQL output labels are interpolated at runtime; they are not "
4526
+ "modeled as DataFrame columns.",
4391
4527
  )
4528
+ for message in summary.notes:
4529
+ self._note(context, node, "CT_SQL_RECOVERED", message)
4392
4530
  for message in summary.diagnostics:
4393
4531
  self._diagnose(context, node, "CT_PARTIAL_SQL", message)
4394
4532
  else:
@@ -4425,6 +4563,7 @@ class _Analyzer:
4425
4563
  fields = ()
4426
4564
  if item.columns:
4427
4565
  fields = tuple(SourceField(name, name) for name in item.columns)
4566
+ references = {}
4428
4567
  complete = True
4429
4568
  if item.output_columns is not None:
4430
4569
  if (
@@ -4448,6 +4587,10 @@ class _Analyzer:
4448
4587
  and len(fields) == len(item.output_columns)
4449
4588
  and not item.reader.endswith("read_sql_table")
4450
4589
  ):
4590
+ references = {
4591
+ name: references.get(original.name, ())
4592
+ for name, original in zip(item.output_columns, fields, strict=True)
4593
+ }
4451
4594
  fields = tuple(
4452
4595
  SourceField(name, original.expression, original.inputs)
4453
4596
  for name, original in zip(item.output_columns, fields, strict=True)
@@ -4464,7 +4607,7 @@ class _Analyzer:
4464
4607
  state.schema = {field.name for field in fields} if complete else None
4465
4608
  expression = self._code(context, node)
4466
4609
  if item.query is not None:
4467
- expression += "\nSQL: " + item.query
4610
+ expression += "\nSQL: " + self._display_sql(item.query)
4468
4611
  expression += "\nSource fields: " + ", ".join(
4469
4612
  f"{field.name} <- {field.expression}" for field in fields
4470
4613
  )
@@ -4494,18 +4637,28 @@ class _Analyzer:
4494
4637
  "only observed labels become source columns.",
4495
4638
  )
4496
4639
  for source_field in seed_fields:
4640
+ value, relation = self._source_field_value(
4641
+ source_field, references.get(source_field.name, ()), context, node
4642
+ )
4497
4643
  self._write(
4498
- state,
4499
- source_field.name,
4644
+ state, source_field.name, value, context, node, "source", operation_kind=relation
4645
+ )
4646
+ if control_references:
4647
+ # WHERE/JOIN/GROUP/ORDER columns select rows; they are not value inputs.
4648
+ self._consume(
4500
4649
  _Value(
4501
- "literal",
4502
- expression=source_field.expression,
4650
+ "series",
4651
+ dependencies=tuple(
4652
+ self._table_column(relation, column, context, node)
4653
+ for relation, column in control_references
4654
+ ),
4503
4655
  evidence="inferred",
4504
4656
  completeness="partial",
4505
4657
  ),
4506
4658
  context,
4507
4659
  node,
4508
- "source",
4660
+ "filter",
4661
+ "SQL row control (WHERE, JOIN, GROUP BY, ORDER BY or window)",
4509
4662
  )
4510
4663
  self.sources.append(
4511
4664
  SourceInfo(
@@ -4517,7 +4670,7 @@ class _Analyzer:
4517
4670
  context.module.name,
4518
4671
  context.scope,
4519
4672
  context.key,
4520
- query=item.query or "",
4673
+ query=self._display_sql(item.query or ""),
4521
4674
  relations=relations,
4522
4675
  fields=fields,
4523
4676
  controls=controls,
@@ -4529,6 +4682,123 @@ class _Analyzer:
4529
4682
  self._seed_source_schema(state, context, node)
4530
4683
  return self._frame_value(state)
4531
4684
 
4685
+ def _source_field_value(
4686
+ self,
4687
+ source_field: SourceField,
4688
+ references: tuple[tuple[str, str], ...],
4689
+ context: _Context,
4690
+ node: ast.AST,
4691
+ ) -> tuple[_Value, str | None]:
4692
+ """A SQL output field's value, reading the physical columns it is built from.
4693
+
4694
+ A field that only selects one column keeps its values (``pass``) or
4695
+ renames them; any other expression derives a new value.
4696
+ """
4697
+ identifiers = tuple(
4698
+ self._table_column(relation, column, context, node)
4699
+ for relation, column in references
4700
+ if PLACEHOLDER_PREFIX not in column
4701
+ )
4702
+ dynamic = len(identifiers) != len(references)
4703
+ if not identifiers:
4704
+ return (
4705
+ _Value(
4706
+ "literal",
4707
+ expression=source_field.expression,
4708
+ evidence="inferred",
4709
+ completeness="partial",
4710
+ ),
4711
+ None,
4712
+ )
4713
+ value = _Value(
4714
+ "series",
4715
+ dependencies=identifiers,
4716
+ expression=source_field.expression,
4717
+ evidence="inferred",
4718
+ completeness="partial",
4719
+ )
4720
+ relation: str | None = None
4721
+ if len(references) == 1 and not dynamic:
4722
+ column = references[0][1]
4723
+ bare = source_field.expression.rsplit(".", 1)[-1].strip('`"[]')
4724
+ if bare == column:
4725
+ relation = "pass" if column == source_field.name else "rename"
4726
+ return value, relation
4727
+
4728
+ def _table_column(self, relation: str, column: str, context: _Context, node: ast.AST) -> str:
4729
+ """The column node of a database relation that SQL text reads.
4730
+
4731
+ A fully named relation is one identity across the project. A relation
4732
+ named by unknown interpolations stays local to its extraction site,
4733
+ because two such names need not denote the same table.
4734
+ """
4735
+ location = self._location(context, node)
4736
+ key = relation
4737
+ if PLACEHOLDER_PREFIX in relation:
4738
+ key = f"{relation}@{location.path}:{location.line}|{context.key}"
4739
+ frame_id = self.table_frames.get(key)
4740
+ if frame_id is None:
4741
+ label = self._display_sql(relation)
4742
+ frame_id = _identifier("table", key)
4743
+ record = Frame(
4744
+ frame_id,
4745
+ label,
4746
+ location,
4747
+ context.module.name,
4748
+ context.scope,
4749
+ context.key,
4750
+ "inferred",
4751
+ "table",
4752
+ )
4753
+ state = _State(record, index_key=record.id)
4754
+ state.history = (
4755
+ self._event(
4756
+ context,
4757
+ node,
4758
+ "source_table",
4759
+ f"Database relation {label} read by SQL",
4760
+ frames=(record.id,),
4761
+ expression=self._code(context, node),
4762
+ evidence="inferred",
4763
+ completeness="partial",
4764
+ ),
4765
+ )
4766
+ self.frame_index[record.id] = len(self.frames)
4767
+ self.frames.append(record)
4768
+ self.states[record.id] = state
4769
+ self.table_frames[key] = frame_id
4770
+ identifier = self.table_columns.get((frame_id, column))
4771
+ if identifier is None:
4772
+ identifier = _identifier("tcol", frame_id, column)
4773
+ label = self.frames[self.frame_index[frame_id]].origin
4774
+ self.columns[identifier] = Column(
4775
+ identifier,
4776
+ frame_id,
4777
+ column,
4778
+ 0,
4779
+ location,
4780
+ context.module.name,
4781
+ context.scope,
4782
+ context.key,
4783
+ "table",
4784
+ "inferred",
4785
+ "partial",
4786
+ )
4787
+ self.column_history[identifier] = self._event(
4788
+ context,
4789
+ node,
4790
+ "source_table",
4791
+ f"Database column {column} of {label}",
4792
+ frames=(frame_id,),
4793
+ outputs=(identifier,),
4794
+ parents=self.states[frame_id].history,
4795
+ expression=self._code(context, node),
4796
+ evidence="inferred",
4797
+ completeness="partial",
4798
+ )
4799
+ self.table_columns[(frame_id, column)] = identifier
4800
+ return identifier
4801
+
4532
4802
  def _matching_source_schema(self, context: _Context, node: ast.AST) -> SourceSchema | None:
4533
4803
  location = self._location(context, node)
4534
4804
  matches = [
@@ -5874,6 +6144,7 @@ class _Analyzer:
5874
6144
  return self.frame_usage[key][1 if loose else 0]
5875
6145
  names: set[str] = set()
5876
6146
  weak: set[str] = set()
6147
+ # Names ruled out as Pandas frames (Spark-only APIs, SQL clients).
5877
6148
  spark: set[str] = set()
5878
6149
 
5879
6150
  class Visitor(ast.NodeVisitor):
@@ -5914,6 +6185,25 @@ class _Analyzer:
5914
6185
  names.add(child.value.id)
5915
6186
  self.generic_visit(child)
5916
6187
 
6188
+ def visit_Call(self, child: ast.Call) -> None:
6189
+ # client.query("SELECT ...") is a SQL client, not DataFrame.query.
6190
+ if (
6191
+ isinstance(child.func, ast.Attribute)
6192
+ and isinstance(child.func.value, ast.Name)
6193
+ and child.func.attr in {"query", "sql", "execute"}
6194
+ and child.args
6195
+ ):
6196
+ first = child.args[0]
6197
+ if isinstance(first, ast.JoinedStr) and first.values:
6198
+ first = first.values[0]
6199
+ if (
6200
+ isinstance(first, ast.Constant)
6201
+ and isinstance(first.value, str)
6202
+ and _Analyzer._looks_like_sql(first.value)
6203
+ ):
6204
+ spark.add(child.func.value.id)
6205
+ self.generic_visit(child)
6206
+
5917
6207
  def visit_FunctionDef(self, child: ast.FunctionDef) -> None:
5918
6208
  pass
5919
6209
 
@@ -5943,15 +6233,12 @@ class _Analyzer:
5943
6233
  arguments: list[_Value],
5944
6234
  keywords: list[tuple[str | None, _Value]],
5945
6235
  values: list[_Value],
6236
+ receiver: _Value | None = None,
5946
6237
  ) -> _Value:
5947
6238
  """An unresolved call result; it may later be used as a DataFrame."""
5948
6239
  callee = ast.unparse(node.func)
5949
- conversion = (
5950
- isinstance(node.func, ast.Attribute)
5951
- and node.func.attr in PANDAS_CONVERSIONS
5952
- and not node.args
5953
- and not node.keywords
5954
- )
6240
+ method = node.func.attr if isinstance(node.func, ast.Attribute) else ""
6241
+ conversion = method in PANDAS_CONVERSIONS and not node.args and not node.keywords
5955
6242
  query = next(
5956
6243
  (
5957
6244
  text
@@ -5960,15 +6247,37 @@ class _Analyzer:
5960
6247
  ),
5961
6248
  None,
5962
6249
  )
6250
+ dialect = _DIALECT_HINTS.get(method) if query is not None else None
6251
+ inner = receiver.opaque if receiver is not None else None
6252
+ if conversion and inner is not None and inner.query is not None:
6253
+ # spark.sql(text).toPandas(), client.query(text).to_dataframe(): the
6254
+ # documented conversion returns the rows the SQL text describes.
6255
+ query, dialect = inner.query, inner.dialect
6256
+ callee = f"{inner.callee}(...).{method}"
5963
6257
  return _Value(
5964
6258
  escaped=self._escaped(values),
5965
- opaque=_Opaque(node, context, callee, query, conversion),
6259
+ opaque=_Opaque(node, context, callee, query, conversion, dialect=dialect),
5966
6260
  )
5967
6261
 
5968
6262
  @staticmethod
5969
6263
  def _looks_like_sql(text: str) -> bool:
5970
- head = text.lstrip().lstrip("(").lstrip().lower()
5971
- return head.startswith(("select ", "select\n", "with ", "with\n"))
6264
+ """A SELECT or WITH statement, possibly after comments or parentheses."""
6265
+ head = text
6266
+ for _ in range(100):
6267
+ head = head.lstrip().lstrip("(").lstrip()
6268
+ if head.startswith("--"):
6269
+ newline = head.find("\n")
6270
+ if newline < 0:
6271
+ return False
6272
+ head = head[newline + 1 :]
6273
+ elif head.startswith("/*"):
6274
+ end = head.find("*/")
6275
+ if end < 0:
6276
+ return False
6277
+ head = head[end + 2 :]
6278
+ else:
6279
+ break
6280
+ return re.match(r"(select|with)\s", head, re.IGNORECASE) is not None
5972
6281
 
5973
6282
  def _promote(self, name: str, value: _Value, context: _Context) -> _Value:
5974
6283
  """Bind an unresolved call result as an inferred-origin DataFrame when used as one."""
@@ -5994,7 +6303,7 @@ class _Analyzer:
5994
6303
  "the extraction explicit.",
5995
6304
  )
5996
6305
  if opaque.query is not None:
5997
- item = Extraction("sql", opaque.callee, query=opaque.query)
6306
+ item = Extraction("sql", opaque.callee, query=opaque.query, dialect=opaque.dialect)
5998
6307
  self._note(
5999
6308
  context,
6000
6309
  node,
@@ -7933,7 +8242,14 @@ def analyze_project(
7933
8242
  }
7934
8243
  if len({item.callable for item in source_contracts}) != len(source_contracts):
7935
8244
  raise ValueError("Source contracts must have unique callable identities")
7936
- analyzer = _Analyzer(modules, ambiguous, source_contracts, source_schemas, settings.observers)
8245
+ analyzer = _Analyzer(
8246
+ modules,
8247
+ ambiguous,
8248
+ source_contracts,
8249
+ source_schemas,
8250
+ settings.observers,
8251
+ settings.sql_dialect,
8252
+ )
7937
8253
  analyzer.run()
7938
8254
  for schema in source_schemas:
7939
8255
  if schema not in analyzer.used_source_schemas: