coltrace 0.1.0.dev7__py3-none-any.whl → 0.1.0.dev8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coltrace/__init__.py +1 -1
- coltrace/analysis.py +362 -46
- coltrace/cli.py +7 -1
- coltrace/compatibility.py +77 -18
- coltrace/exporters.py +717 -711
- coltrace/graph.py +93 -18
- coltrace/journey.py +18 -0
- coltrace/lineage_models.py +9 -2
- coltrace/models.py +1 -1
- coltrace/scanner.py +20 -2
- coltrace/sql_sources.py +561 -355
- {coltrace-0.1.0.dev7.dist-info → coltrace-0.1.0.dev8.dist-info}/METADATA +12 -7
- coltrace-0.1.0.dev8.dist-info/RECORD +30 -0
- coltrace-0.1.0.dev7.dist-info/RECORD +0 -30
- {coltrace-0.1.0.dev7.dist-info → coltrace-0.1.0.dev8.dist-info}/WHEEL +0 -0
- {coltrace-0.1.0.dev7.dist-info → coltrace-0.1.0.dev8.dist-info}/entry_points.txt +0 -0
- {coltrace-0.1.0.dev7.dist-info → coltrace-0.1.0.dev8.dist-info}/licenses/LICENSE +0 -0
- {coltrace-0.1.0.dev7.dist-info → coltrace-0.1.0.dev8.dist-info}/top_level.txt +0 -0
coltrace/__init__.py
CHANGED
|
@@ -7,7 +7,7 @@ from .models import ScanReport
|
|
|
7
7
|
from .scanner import ScanConfig, scan_project
|
|
8
8
|
from .source_models import SourceContract, SourceSchema, schema_from_columns
|
|
9
9
|
|
|
10
|
-
__version__ = "0.1.0.
|
|
10
|
+
__version__ = "0.1.0.dev8"
|
|
11
11
|
__all__ = [
|
|
12
12
|
"LineageReport",
|
|
13
13
|
"ScanConfig",
|
coltrace/analysis.py
CHANGED
|
@@ -12,6 +12,7 @@ import fnmatch
|
|
|
12
12
|
import hashlib
|
|
13
13
|
import io
|
|
14
14
|
import operator
|
|
15
|
+
import re
|
|
15
16
|
import sys
|
|
16
17
|
import tokenize
|
|
17
18
|
import warnings
|
|
@@ -136,6 +137,10 @@ class _Value:
|
|
|
136
137
|
|
|
137
138
|
|
|
138
139
|
PLACEHOLDER = "__coltrace_param_{}__"
|
|
140
|
+
PLACEHOLDER_PREFIX = "__coltrace_param_"
|
|
141
|
+
_PLACEHOLDER_NAME = re.compile(r"__coltrace_param_\d+__")
|
|
142
|
+
# Method names whose receiver dialect is a useful SQL parsing hint.
|
|
143
|
+
_DIALECT_HINTS = {"sql": "databricks", "query": "bigquery", "query_and_wait": "bigquery"}
|
|
139
144
|
|
|
140
145
|
# Library constants: scalar literals, not objects that could hold data.
|
|
141
146
|
CONSTANT_SYMBOLS = frozenset(
|
|
@@ -219,6 +224,8 @@ class _Opaque:
|
|
|
219
224
|
conversion: bool = False
|
|
220
225
|
# Labels declared for the resulting DataFrame, such as columns=[...].
|
|
221
226
|
columns: tuple[str, ...] | None = None
|
|
227
|
+
# SQL dialect suggested by the call shape; parsing still falls back.
|
|
228
|
+
dialect: str | None = None
|
|
222
229
|
|
|
223
230
|
|
|
224
231
|
@dataclass
|
|
@@ -382,8 +389,16 @@ class _Analyzer:
|
|
|
382
389
|
source_contracts: tuple[SourceContract, ...] = (),
|
|
383
390
|
source_schemas: tuple[SourceSchema, ...] = (),
|
|
384
391
|
observers: tuple[str, ...] = (),
|
|
392
|
+
sql_dialect: str | None = None,
|
|
385
393
|
) -> None:
|
|
386
394
|
self.observers = observers
|
|
395
|
+
self.sql_dialect = sql_dialect
|
|
396
|
+
# Unknown interpolations in SQL text: stable names per source position.
|
|
397
|
+
self.placeholders: dict[str, str] = {}
|
|
398
|
+
self.placeholder_sites: dict[tuple[str, int, int, int, int], str] = {}
|
|
399
|
+
# Database relations read by SQL sources, and their physical columns.
|
|
400
|
+
self.table_frames: dict[str, str] = {}
|
|
401
|
+
self.table_columns: dict[tuple[str, str], str] = {}
|
|
387
402
|
self.modules = modules
|
|
388
403
|
self.ambiguous = ambiguous
|
|
389
404
|
self.frames: list[Frame] = []
|
|
@@ -693,6 +708,7 @@ class _Analyzer:
|
|
|
693
708
|
context: _Context,
|
|
694
709
|
node: ast.AST,
|
|
695
710
|
kind: str = "derive",
|
|
711
|
+
operation_kind: str | None = None,
|
|
696
712
|
) -> str:
|
|
697
713
|
value = self._refresh(value)
|
|
698
714
|
state.revision += 1
|
|
@@ -746,7 +762,9 @@ class _Analyzer:
|
|
|
746
762
|
state.removed.discard(name)
|
|
747
763
|
if state.schema is not None:
|
|
748
764
|
state.schema.add(name)
|
|
749
|
-
operation = self._operation(
|
|
765
|
+
operation = self._operation(
|
|
766
|
+
context, node, operation_kind or kind, column.id, evidence, value.completeness
|
|
767
|
+
)
|
|
750
768
|
for dependency in dict.fromkeys(value.dependencies):
|
|
751
769
|
self.edges.append(Edge(dependency, operation.id, "read", evidence))
|
|
752
770
|
self.edges.append(Edge(operation.id, column.id, "write", evidence))
|
|
@@ -1439,7 +1457,14 @@ class _Analyzer:
|
|
|
1439
1457
|
identifiers.extend(self.columns[column].dataframe_id for column in value.dependencies)
|
|
1440
1458
|
return tuple(dict.fromkeys(identifiers))
|
|
1441
1459
|
|
|
1442
|
-
def _consume(
|
|
1460
|
+
def _consume(
|
|
1461
|
+
self,
|
|
1462
|
+
value: _Value,
|
|
1463
|
+
context: _Context,
|
|
1464
|
+
node: ast.AST,
|
|
1465
|
+
kind: str,
|
|
1466
|
+
label: str | None = None,
|
|
1467
|
+
) -> str | None:
|
|
1443
1468
|
dependencies = value.dependencies
|
|
1444
1469
|
if value.frame is not None:
|
|
1445
1470
|
dependencies = tuple(value.frame.current.values())
|
|
@@ -1467,7 +1492,7 @@ class _Analyzer:
|
|
|
1467
1492
|
context,
|
|
1468
1493
|
node,
|
|
1469
1494
|
event_kind,
|
|
1470
|
-
kind.replace("_", " "),
|
|
1495
|
+
label or kind.replace("_", " "),
|
|
1471
1496
|
frames=frames,
|
|
1472
1497
|
inputs=dependencies,
|
|
1473
1498
|
parents=self._parents(value)
|
|
@@ -1686,18 +1711,18 @@ class _Analyzer:
|
|
|
1686
1711
|
return folded
|
|
1687
1712
|
if (
|
|
1688
1713
|
isinstance(node.op, ast.Add)
|
|
1689
|
-
and all(value.kind == "literal" for value in values)
|
|
1714
|
+
and all(value.kind == "literal" or self._plain_unknown(value) for value in values)
|
|
1690
1715
|
and (self._text(left) is not None or self._text(right) is not None)
|
|
1691
1716
|
):
|
|
1692
1717
|
# A partially known string keeps its known text for SQL parsing.
|
|
1693
1718
|
return _Value(
|
|
1694
1719
|
"literal",
|
|
1695
1720
|
completeness="complete",
|
|
1696
|
-
template=(self._text(left) or
|
|
1697
|
-
+ (self._text(right) or
|
|
1721
|
+
template=(self._text(left) or self._placeholder(node.left, context))
|
|
1722
|
+
+ (self._text(right) or self._placeholder(node.right, context)),
|
|
1698
1723
|
)
|
|
1699
1724
|
if isinstance(node.op, ast.Mod) and left.known and isinstance(left.constant, str):
|
|
1700
|
-
return
|
|
1725
|
+
return self._percent_format(node, left.constant, right, context)
|
|
1701
1726
|
if isinstance(node.op, ast.Add):
|
|
1702
1727
|
first, second = self._elements(left), self._elements(right)
|
|
1703
1728
|
if (
|
|
@@ -1721,6 +1746,65 @@ class _Analyzer:
|
|
|
1721
1746
|
)
|
|
1722
1747
|
return self._degraded(values, kind=kind)
|
|
1723
1748
|
|
|
1749
|
+
def _placeholder(self, node: ast.AST, context: _Context) -> str:
|
|
1750
|
+
"""A stable SQL-safe name for one unknown interpolation, labeled by its code."""
|
|
1751
|
+
key = (
|
|
1752
|
+
context.module.name,
|
|
1753
|
+
getattr(node, "lineno", 0),
|
|
1754
|
+
getattr(node, "col_offset", 0),
|
|
1755
|
+
getattr(node, "end_lineno", 0),
|
|
1756
|
+
getattr(node, "end_col_offset", 0),
|
|
1757
|
+
)
|
|
1758
|
+
name = self.placeholder_sites.get(key)
|
|
1759
|
+
if name is None:
|
|
1760
|
+
name = PLACEHOLDER.format(len(self.placeholders) + 1)
|
|
1761
|
+
label = " ".join(self._code(context, node).split())
|
|
1762
|
+
if len(label) > 60:
|
|
1763
|
+
label = label[:57] + "..."
|
|
1764
|
+
self.placeholders[name] = "{" + label + "}"
|
|
1765
|
+
self.placeholder_sites[key] = name
|
|
1766
|
+
return name
|
|
1767
|
+
|
|
1768
|
+
def _display_sql(self, text: str) -> str:
|
|
1769
|
+
"""Show unknown interpolations as their Python expression, e.g. ``{table}``."""
|
|
1770
|
+
return _PLACEHOLDER_NAME.sub(
|
|
1771
|
+
lambda match: self.placeholders.get(match.group(0), "{?}"), text
|
|
1772
|
+
)
|
|
1773
|
+
|
|
1774
|
+
@staticmethod
|
|
1775
|
+
def _plain_unknown(value: _Value) -> bool:
|
|
1776
|
+
"""An unknown scalar-like value: not a frame, column, object or collection."""
|
|
1777
|
+
return (
|
|
1778
|
+
value.kind == "unknown"
|
|
1779
|
+
and value.frame is None
|
|
1780
|
+
and not value.dependencies
|
|
1781
|
+
and value.instance is None
|
|
1782
|
+
and value.function is None
|
|
1783
|
+
and value.items is None
|
|
1784
|
+
and value.members is None
|
|
1785
|
+
and value.opaque is None
|
|
1786
|
+
)
|
|
1787
|
+
|
|
1788
|
+
def _percent_format(
|
|
1789
|
+
self, node: ast.BinOp, text: str, right: _Value, context: _Context
|
|
1790
|
+
) -> _Value:
|
|
1791
|
+
"""``"..." % values``: fold known values, otherwise keep a SQL template."""
|
|
1792
|
+
if right.known and isinstance(right.constant, (tuple, dict, str, int, float, bool)):
|
|
1793
|
+
try:
|
|
1794
|
+
result = text % right.constant
|
|
1795
|
+
except (TypeError, ValueError, KeyError, OverflowError):
|
|
1796
|
+
return _Value("literal", completeness="complete")
|
|
1797
|
+
if len(result) <= 1_000_000:
|
|
1798
|
+
return self._literal_value(result)
|
|
1799
|
+
return _Value("literal", completeness="complete")
|
|
1800
|
+
name = self._placeholder(node.right, context)
|
|
1801
|
+
template = re.sub(
|
|
1802
|
+
r"%%|%(\([^)]*\))?[-#0 +]*\d*(\.\d+)?[sdifrxXeEgGc]",
|
|
1803
|
+
lambda match: "%" if match.group(0) == "%%" else name,
|
|
1804
|
+
text,
|
|
1805
|
+
)
|
|
1806
|
+
return _Value("literal", completeness="complete", template=template)
|
|
1807
|
+
|
|
1724
1808
|
def _text(self, value: _Value) -> str | None:
|
|
1725
1809
|
"""String text, with placeholders for unknown interpolated parts."""
|
|
1726
1810
|
if value.known and isinstance(value.constant, str):
|
|
@@ -1868,7 +1952,7 @@ class _Analyzer:
|
|
|
1868
1952
|
pattern.append(
|
|
1869
1953
|
item.template
|
|
1870
1954
|
if item.template is not None and formatted.format_spec is None
|
|
1871
|
-
else
|
|
1955
|
+
else self._placeholder(formatted.value, context)
|
|
1872
1956
|
)
|
|
1873
1957
|
if formatted.format_spec is not None:
|
|
1874
1958
|
values.append(self._expr(formatted.format_spec, context))
|
|
@@ -1981,7 +2065,9 @@ class _Analyzer:
|
|
|
1981
2065
|
node: ast.ListComp | ast.SetComp | ast.GeneratorExp | ast.DictComp,
|
|
1982
2066
|
context: _Context,
|
|
1983
2067
|
) -> _Value | None:
|
|
1984
|
-
|
|
2068
|
+
# A generator over known constants evaluates like a list: its consumer
|
|
2069
|
+
# (join, tuple, any, ...) then folds statically.
|
|
2070
|
+
if len(node.generators) != 1:
|
|
1985
2071
|
return None
|
|
1986
2072
|
generator = node.generators[0]
|
|
1987
2073
|
if generator.is_async or not isinstance(generator.target, (ast.Name, ast.Tuple)):
|
|
@@ -1999,6 +2085,10 @@ class _Analyzer:
|
|
|
1999
2085
|
for call in calls
|
|
2000
2086
|
):
|
|
2001
2087
|
return None
|
|
2088
|
+
if isinstance(node, ast.GeneratorExp) and calls:
|
|
2089
|
+
# Only pure element expressions (names, constants, f-strings) are
|
|
2090
|
+
# evaluated per item; anything else keeps the one-element summary.
|
|
2091
|
+
return None
|
|
2002
2092
|
iterable = self._expr(generator.iter, context)
|
|
2003
2093
|
if (
|
|
2004
2094
|
not iterable.known
|
|
@@ -2146,7 +2236,7 @@ class _Analyzer:
|
|
|
2146
2236
|
and node.func.attr == "format"
|
|
2147
2237
|
and receiver.template is not None
|
|
2148
2238
|
):
|
|
2149
|
-
formatted = self._format(receiver.template, arguments, keywords)
|
|
2239
|
+
formatted = self._format(receiver.template, arguments, keywords, node, context)
|
|
2150
2240
|
if formatted is not None:
|
|
2151
2241
|
text = (
|
|
2152
2242
|
formatted.template if formatted.template is not None else formatted.constant
|
|
@@ -2319,7 +2409,7 @@ class _Analyzer:
|
|
|
2319
2409
|
"Call effects and return provenance are unresolved; affected state is invalidated.",
|
|
2320
2410
|
)
|
|
2321
2411
|
self._invalidate(values, context, node)
|
|
2322
|
-
return self._opaque_result(node, context, function, arguments, keywords, values)
|
|
2412
|
+
return self._opaque_result(node, context, function, arguments, keywords, values, receiver)
|
|
2323
2413
|
|
|
2324
2414
|
def _column_label(
|
|
2325
2415
|
self, node: ast.Subscript, context: _Context
|
|
@@ -2837,7 +2927,7 @@ class _Analyzer:
|
|
|
2837
2927
|
allowed = CONSTANT_METHODS.get(type(receiver.constant), frozenset())
|
|
2838
2928
|
if method not in allowed:
|
|
2839
2929
|
if method == "format" and isinstance(receiver.constant, str):
|
|
2840
|
-
return self._format(receiver.constant, arguments, keywords)
|
|
2930
|
+
return self._format(receiver.constant, arguments, keywords, node, context)
|
|
2841
2931
|
return None
|
|
2842
2932
|
try:
|
|
2843
2933
|
result = getattr(receiver.constant, method)(*args, **kwargs)
|
|
@@ -2849,9 +2939,13 @@ class _Analyzer:
|
|
|
2849
2939
|
return None
|
|
2850
2940
|
return self._literal_value(result)
|
|
2851
2941
|
|
|
2852
|
-
@staticmethod
|
|
2853
2942
|
def _format(
|
|
2854
|
-
|
|
2943
|
+
self,
|
|
2944
|
+
text: str,
|
|
2945
|
+
arguments: list[_Value],
|
|
2946
|
+
keywords: list[tuple[str | None, _Value]],
|
|
2947
|
+
node: ast.Call,
|
|
2948
|
+
context: _Context,
|
|
2855
2949
|
) -> _Value | None:
|
|
2856
2950
|
"""``str.format`` with simple field names; unknown values become placeholders."""
|
|
2857
2951
|
import string
|
|
@@ -2864,10 +2958,17 @@ class _Analyzer:
|
|
|
2864
2958
|
return None
|
|
2865
2959
|
known = all(item.known for item in arguments) and all(item.known for _, item in keywords)
|
|
2866
2960
|
|
|
2961
|
+
sources: dict[object, ast.AST] = dict(enumerate(node.args))
|
|
2962
|
+
sources.update(
|
|
2963
|
+
(keyword.arg, keyword.value) for keyword in node.keywords if keyword.arg is not None
|
|
2964
|
+
)
|
|
2965
|
+
|
|
2867
2966
|
def shown(item: _Value, index: object) -> object:
|
|
2868
2967
|
if item.known and isinstance(item.constant, (str, int, float, bool)):
|
|
2869
2968
|
return item.constant
|
|
2870
|
-
|
|
2969
|
+
if item.template:
|
|
2970
|
+
return item.template
|
|
2971
|
+
return self._placeholder(sources.get(index, node), context)
|
|
2871
2972
|
|
|
2872
2973
|
try:
|
|
2873
2974
|
result = text.format(
|
|
@@ -2877,7 +2978,7 @@ class _Analyzer:
|
|
|
2877
2978
|
except (IndexError, KeyError, ValueError):
|
|
2878
2979
|
return None
|
|
2879
2980
|
if known:
|
|
2880
|
-
return
|
|
2981
|
+
return self._literal_value(result)
|
|
2881
2982
|
return _Value("literal", completeness="complete", template=result)
|
|
2882
2983
|
|
|
2883
2984
|
def _derived_call(
|
|
@@ -4361,34 +4462,71 @@ class _Analyzer:
|
|
|
4361
4462
|
relations: tuple[str, ...] = (item.relation,) if item.relation else ()
|
|
4362
4463
|
controls: tuple[str, ...] = ()
|
|
4363
4464
|
complete = False
|
|
4465
|
+
# Physical (relation, column) inputs per output field, and SQL controls.
|
|
4466
|
+
references: dict[str, tuple[tuple[str, str], ...]] = {}
|
|
4467
|
+
control_references: tuple[tuple[str, str], ...] = ()
|
|
4364
4468
|
if item.kind == "sql":
|
|
4365
4469
|
if item.query is not None:
|
|
4366
4470
|
from .sql_sources import summarize_sql
|
|
4367
4471
|
|
|
4368
4472
|
try:
|
|
4369
|
-
summary = summarize_sql(
|
|
4473
|
+
summary = summarize_sql(
|
|
4474
|
+
item.query,
|
|
4475
|
+
item.dialect or self.sql_dialect,
|
|
4476
|
+
fallback=True,
|
|
4477
|
+
placeholder=PLACEHOLDER_PREFIX,
|
|
4478
|
+
)
|
|
4370
4479
|
except ValueError as error:
|
|
4371
4480
|
self._diagnose(context, node, "CT_UNRESOLVED_SQL", str(error))
|
|
4372
4481
|
else:
|
|
4373
|
-
marker =
|
|
4482
|
+
marker = PLACEHOLDER_PREFIX
|
|
4483
|
+
static = [field for field in summary.fields if marker not in field.name]
|
|
4374
4484
|
fields = tuple(
|
|
4375
|
-
SourceField(
|
|
4376
|
-
|
|
4377
|
-
|
|
4485
|
+
SourceField(
|
|
4486
|
+
field.name,
|
|
4487
|
+
self._display_sql(field.expression),
|
|
4488
|
+
tuple(self._display_sql(name) for name in field.inputs),
|
|
4489
|
+
)
|
|
4490
|
+
for field in static
|
|
4378
4491
|
)
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
|
|
4492
|
+
references = {field.name: field.references for field in static}
|
|
4493
|
+
control_references = tuple(
|
|
4494
|
+
reference
|
|
4495
|
+
for reference in summary.control_references
|
|
4496
|
+
if marker not in reference[1]
|
|
4383
4497
|
)
|
|
4384
|
-
|
|
4498
|
+
relations = tuple(self._display_sql(name) for name in summary.relations)
|
|
4499
|
+
controls = tuple(
|
|
4500
|
+
self._display_sql(name)
|
|
4501
|
+
for name in summary.controls
|
|
4502
|
+
if marker not in name.rsplit(".", 1)[-1]
|
|
4503
|
+
)
|
|
4504
|
+
complete = summary.complete and len(static) == len(summary.fields)
|
|
4505
|
+
interpolations = list(dict.fromkeys(_PLACEHOLDER_NAME.findall(item.query)))
|
|
4506
|
+
if interpolations:
|
|
4507
|
+
shown = ", ".join(
|
|
4508
|
+
self.placeholders.get(name, "{?}") for name in interpolations[:4]
|
|
4509
|
+
)
|
|
4510
|
+
more = len(interpolations) - 4
|
|
4385
4511
|
self._note(
|
|
4512
|
+
context,
|
|
4513
|
+
node,
|
|
4514
|
+
"CT_SQL_INTERPOLATION",
|
|
4515
|
+
f"SQL text has {len(interpolations)} interpolation(s) that are not "
|
|
4516
|
+
f"statically known ({shown}{f', +{more}' if more > 0 else ''}). "
|
|
4517
|
+
"The known text is parsed; relations and columns named by them "
|
|
4518
|
+
"are shown as {expression} and fields reading them are partial.",
|
|
4519
|
+
)
|
|
4520
|
+
if len(static) != len(summary.fields):
|
|
4521
|
+
self._diagnose(
|
|
4386
4522
|
context,
|
|
4387
4523
|
node,
|
|
4388
4524
|
"CT_PARTIAL_SQL",
|
|
4389
|
-
"SQL
|
|
4390
|
-
"
|
|
4525
|
+
"Some SQL output labels are interpolated at runtime; they are not "
|
|
4526
|
+
"modeled as DataFrame columns.",
|
|
4391
4527
|
)
|
|
4528
|
+
for message in summary.notes:
|
|
4529
|
+
self._note(context, node, "CT_SQL_RECOVERED", message)
|
|
4392
4530
|
for message in summary.diagnostics:
|
|
4393
4531
|
self._diagnose(context, node, "CT_PARTIAL_SQL", message)
|
|
4394
4532
|
else:
|
|
@@ -4425,6 +4563,7 @@ class _Analyzer:
|
|
|
4425
4563
|
fields = ()
|
|
4426
4564
|
if item.columns:
|
|
4427
4565
|
fields = tuple(SourceField(name, name) for name in item.columns)
|
|
4566
|
+
references = {}
|
|
4428
4567
|
complete = True
|
|
4429
4568
|
if item.output_columns is not None:
|
|
4430
4569
|
if (
|
|
@@ -4448,6 +4587,10 @@ class _Analyzer:
|
|
|
4448
4587
|
and len(fields) == len(item.output_columns)
|
|
4449
4588
|
and not item.reader.endswith("read_sql_table")
|
|
4450
4589
|
):
|
|
4590
|
+
references = {
|
|
4591
|
+
name: references.get(original.name, ())
|
|
4592
|
+
for name, original in zip(item.output_columns, fields, strict=True)
|
|
4593
|
+
}
|
|
4451
4594
|
fields = tuple(
|
|
4452
4595
|
SourceField(name, original.expression, original.inputs)
|
|
4453
4596
|
for name, original in zip(item.output_columns, fields, strict=True)
|
|
@@ -4464,7 +4607,7 @@ class _Analyzer:
|
|
|
4464
4607
|
state.schema = {field.name for field in fields} if complete else None
|
|
4465
4608
|
expression = self._code(context, node)
|
|
4466
4609
|
if item.query is not None:
|
|
4467
|
-
expression += "\nSQL: " + item.query
|
|
4610
|
+
expression += "\nSQL: " + self._display_sql(item.query)
|
|
4468
4611
|
expression += "\nSource fields: " + ", ".join(
|
|
4469
4612
|
f"{field.name} <- {field.expression}" for field in fields
|
|
4470
4613
|
)
|
|
@@ -4494,18 +4637,28 @@ class _Analyzer:
|
|
|
4494
4637
|
"only observed labels become source columns.",
|
|
4495
4638
|
)
|
|
4496
4639
|
for source_field in seed_fields:
|
|
4640
|
+
value, relation = self._source_field_value(
|
|
4641
|
+
source_field, references.get(source_field.name, ()), context, node
|
|
4642
|
+
)
|
|
4497
4643
|
self._write(
|
|
4498
|
-
state,
|
|
4499
|
-
|
|
4644
|
+
state, source_field.name, value, context, node, "source", operation_kind=relation
|
|
4645
|
+
)
|
|
4646
|
+
if control_references:
|
|
4647
|
+
# WHERE/JOIN/GROUP/ORDER columns select rows; they are not value inputs.
|
|
4648
|
+
self._consume(
|
|
4500
4649
|
_Value(
|
|
4501
|
-
"
|
|
4502
|
-
|
|
4650
|
+
"series",
|
|
4651
|
+
dependencies=tuple(
|
|
4652
|
+
self._table_column(relation, column, context, node)
|
|
4653
|
+
for relation, column in control_references
|
|
4654
|
+
),
|
|
4503
4655
|
evidence="inferred",
|
|
4504
4656
|
completeness="partial",
|
|
4505
4657
|
),
|
|
4506
4658
|
context,
|
|
4507
4659
|
node,
|
|
4508
|
-
"
|
|
4660
|
+
"filter",
|
|
4661
|
+
"SQL row control (WHERE, JOIN, GROUP BY, ORDER BY or window)",
|
|
4509
4662
|
)
|
|
4510
4663
|
self.sources.append(
|
|
4511
4664
|
SourceInfo(
|
|
@@ -4517,7 +4670,7 @@ class _Analyzer:
|
|
|
4517
4670
|
context.module.name,
|
|
4518
4671
|
context.scope,
|
|
4519
4672
|
context.key,
|
|
4520
|
-
query=item.query or "",
|
|
4673
|
+
query=self._display_sql(item.query or ""),
|
|
4521
4674
|
relations=relations,
|
|
4522
4675
|
fields=fields,
|
|
4523
4676
|
controls=controls,
|
|
@@ -4529,6 +4682,123 @@ class _Analyzer:
|
|
|
4529
4682
|
self._seed_source_schema(state, context, node)
|
|
4530
4683
|
return self._frame_value(state)
|
|
4531
4684
|
|
|
4685
|
+
def _source_field_value(
|
|
4686
|
+
self,
|
|
4687
|
+
source_field: SourceField,
|
|
4688
|
+
references: tuple[tuple[str, str], ...],
|
|
4689
|
+
context: _Context,
|
|
4690
|
+
node: ast.AST,
|
|
4691
|
+
) -> tuple[_Value, str | None]:
|
|
4692
|
+
"""A SQL output field's value, reading the physical columns it is built from.
|
|
4693
|
+
|
|
4694
|
+
A field that only selects one column keeps its values (``pass``) or
|
|
4695
|
+
renames them; any other expression derives a new value.
|
|
4696
|
+
"""
|
|
4697
|
+
identifiers = tuple(
|
|
4698
|
+
self._table_column(relation, column, context, node)
|
|
4699
|
+
for relation, column in references
|
|
4700
|
+
if PLACEHOLDER_PREFIX not in column
|
|
4701
|
+
)
|
|
4702
|
+
dynamic = len(identifiers) != len(references)
|
|
4703
|
+
if not identifiers:
|
|
4704
|
+
return (
|
|
4705
|
+
_Value(
|
|
4706
|
+
"literal",
|
|
4707
|
+
expression=source_field.expression,
|
|
4708
|
+
evidence="inferred",
|
|
4709
|
+
completeness="partial",
|
|
4710
|
+
),
|
|
4711
|
+
None,
|
|
4712
|
+
)
|
|
4713
|
+
value = _Value(
|
|
4714
|
+
"series",
|
|
4715
|
+
dependencies=identifiers,
|
|
4716
|
+
expression=source_field.expression,
|
|
4717
|
+
evidence="inferred",
|
|
4718
|
+
completeness="partial",
|
|
4719
|
+
)
|
|
4720
|
+
relation: str | None = None
|
|
4721
|
+
if len(references) == 1 and not dynamic:
|
|
4722
|
+
column = references[0][1]
|
|
4723
|
+
bare = source_field.expression.rsplit(".", 1)[-1].strip('`"[]')
|
|
4724
|
+
if bare == column:
|
|
4725
|
+
relation = "pass" if column == source_field.name else "rename"
|
|
4726
|
+
return value, relation
|
|
4727
|
+
|
|
4728
|
+
def _table_column(self, relation: str, column: str, context: _Context, node: ast.AST) -> str:
|
|
4729
|
+
"""The column node of a database relation that SQL text reads.
|
|
4730
|
+
|
|
4731
|
+
A fully named relation is one identity across the project. A relation
|
|
4732
|
+
named by unknown interpolations stays local to its extraction site,
|
|
4733
|
+
because two such names need not denote the same table.
|
|
4734
|
+
"""
|
|
4735
|
+
location = self._location(context, node)
|
|
4736
|
+
key = relation
|
|
4737
|
+
if PLACEHOLDER_PREFIX in relation:
|
|
4738
|
+
key = f"{relation}@{location.path}:{location.line}|{context.key}"
|
|
4739
|
+
frame_id = self.table_frames.get(key)
|
|
4740
|
+
if frame_id is None:
|
|
4741
|
+
label = self._display_sql(relation)
|
|
4742
|
+
frame_id = _identifier("table", key)
|
|
4743
|
+
record = Frame(
|
|
4744
|
+
frame_id,
|
|
4745
|
+
label,
|
|
4746
|
+
location,
|
|
4747
|
+
context.module.name,
|
|
4748
|
+
context.scope,
|
|
4749
|
+
context.key,
|
|
4750
|
+
"inferred",
|
|
4751
|
+
"table",
|
|
4752
|
+
)
|
|
4753
|
+
state = _State(record, index_key=record.id)
|
|
4754
|
+
state.history = (
|
|
4755
|
+
self._event(
|
|
4756
|
+
context,
|
|
4757
|
+
node,
|
|
4758
|
+
"source_table",
|
|
4759
|
+
f"Database relation {label} read by SQL",
|
|
4760
|
+
frames=(record.id,),
|
|
4761
|
+
expression=self._code(context, node),
|
|
4762
|
+
evidence="inferred",
|
|
4763
|
+
completeness="partial",
|
|
4764
|
+
),
|
|
4765
|
+
)
|
|
4766
|
+
self.frame_index[record.id] = len(self.frames)
|
|
4767
|
+
self.frames.append(record)
|
|
4768
|
+
self.states[record.id] = state
|
|
4769
|
+
self.table_frames[key] = frame_id
|
|
4770
|
+
identifier = self.table_columns.get((frame_id, column))
|
|
4771
|
+
if identifier is None:
|
|
4772
|
+
identifier = _identifier("tcol", frame_id, column)
|
|
4773
|
+
label = self.frames[self.frame_index[frame_id]].origin
|
|
4774
|
+
self.columns[identifier] = Column(
|
|
4775
|
+
identifier,
|
|
4776
|
+
frame_id,
|
|
4777
|
+
column,
|
|
4778
|
+
0,
|
|
4779
|
+
location,
|
|
4780
|
+
context.module.name,
|
|
4781
|
+
context.scope,
|
|
4782
|
+
context.key,
|
|
4783
|
+
"table",
|
|
4784
|
+
"inferred",
|
|
4785
|
+
"partial",
|
|
4786
|
+
)
|
|
4787
|
+
self.column_history[identifier] = self._event(
|
|
4788
|
+
context,
|
|
4789
|
+
node,
|
|
4790
|
+
"source_table",
|
|
4791
|
+
f"Database column {column} of {label}",
|
|
4792
|
+
frames=(frame_id,),
|
|
4793
|
+
outputs=(identifier,),
|
|
4794
|
+
parents=self.states[frame_id].history,
|
|
4795
|
+
expression=self._code(context, node),
|
|
4796
|
+
evidence="inferred",
|
|
4797
|
+
completeness="partial",
|
|
4798
|
+
)
|
|
4799
|
+
self.table_columns[(frame_id, column)] = identifier
|
|
4800
|
+
return identifier
|
|
4801
|
+
|
|
4532
4802
|
def _matching_source_schema(self, context: _Context, node: ast.AST) -> SourceSchema | None:
|
|
4533
4803
|
location = self._location(context, node)
|
|
4534
4804
|
matches = [
|
|
@@ -5874,6 +6144,7 @@ class _Analyzer:
|
|
|
5874
6144
|
return self.frame_usage[key][1 if loose else 0]
|
|
5875
6145
|
names: set[str] = set()
|
|
5876
6146
|
weak: set[str] = set()
|
|
6147
|
+
# Names ruled out as Pandas frames (Spark-only APIs, SQL clients).
|
|
5877
6148
|
spark: set[str] = set()
|
|
5878
6149
|
|
|
5879
6150
|
class Visitor(ast.NodeVisitor):
|
|
@@ -5914,6 +6185,25 @@ class _Analyzer:
|
|
|
5914
6185
|
names.add(child.value.id)
|
|
5915
6186
|
self.generic_visit(child)
|
|
5916
6187
|
|
|
6188
|
+
def visit_Call(self, child: ast.Call) -> None:
|
|
6189
|
+
# client.query("SELECT ...") is a SQL client, not DataFrame.query.
|
|
6190
|
+
if (
|
|
6191
|
+
isinstance(child.func, ast.Attribute)
|
|
6192
|
+
and isinstance(child.func.value, ast.Name)
|
|
6193
|
+
and child.func.attr in {"query", "sql", "execute"}
|
|
6194
|
+
and child.args
|
|
6195
|
+
):
|
|
6196
|
+
first = child.args[0]
|
|
6197
|
+
if isinstance(first, ast.JoinedStr) and first.values:
|
|
6198
|
+
first = first.values[0]
|
|
6199
|
+
if (
|
|
6200
|
+
isinstance(first, ast.Constant)
|
|
6201
|
+
and isinstance(first.value, str)
|
|
6202
|
+
and _Analyzer._looks_like_sql(first.value)
|
|
6203
|
+
):
|
|
6204
|
+
spark.add(child.func.value.id)
|
|
6205
|
+
self.generic_visit(child)
|
|
6206
|
+
|
|
5917
6207
|
def visit_FunctionDef(self, child: ast.FunctionDef) -> None:
|
|
5918
6208
|
pass
|
|
5919
6209
|
|
|
@@ -5943,15 +6233,12 @@ class _Analyzer:
|
|
|
5943
6233
|
arguments: list[_Value],
|
|
5944
6234
|
keywords: list[tuple[str | None, _Value]],
|
|
5945
6235
|
values: list[_Value],
|
|
6236
|
+
receiver: _Value | None = None,
|
|
5946
6237
|
) -> _Value:
|
|
5947
6238
|
"""An unresolved call result; it may later be used as a DataFrame."""
|
|
5948
6239
|
callee = ast.unparse(node.func)
|
|
5949
|
-
|
|
5950
|
-
|
|
5951
|
-
and node.func.attr in PANDAS_CONVERSIONS
|
|
5952
|
-
and not node.args
|
|
5953
|
-
and not node.keywords
|
|
5954
|
-
)
|
|
6240
|
+
method = node.func.attr if isinstance(node.func, ast.Attribute) else ""
|
|
6241
|
+
conversion = method in PANDAS_CONVERSIONS and not node.args and not node.keywords
|
|
5955
6242
|
query = next(
|
|
5956
6243
|
(
|
|
5957
6244
|
text
|
|
@@ -5960,15 +6247,37 @@ class _Analyzer:
|
|
|
5960
6247
|
),
|
|
5961
6248
|
None,
|
|
5962
6249
|
)
|
|
6250
|
+
dialect = _DIALECT_HINTS.get(method) if query is not None else None
|
|
6251
|
+
inner = receiver.opaque if receiver is not None else None
|
|
6252
|
+
if conversion and inner is not None and inner.query is not None:
|
|
6253
|
+
# spark.sql(text).toPandas(), client.query(text).to_dataframe(): the
|
|
6254
|
+
# documented conversion returns the rows the SQL text describes.
|
|
6255
|
+
query, dialect = inner.query, inner.dialect
|
|
6256
|
+
callee = f"{inner.callee}(...).{method}"
|
|
5963
6257
|
return _Value(
|
|
5964
6258
|
escaped=self._escaped(values),
|
|
5965
|
-
opaque=_Opaque(node, context, callee, query, conversion),
|
|
6259
|
+
opaque=_Opaque(node, context, callee, query, conversion, dialect=dialect),
|
|
5966
6260
|
)
|
|
5967
6261
|
|
|
5968
6262
|
@staticmethod
|
|
5969
6263
|
def _looks_like_sql(text: str) -> bool:
|
|
5970
|
-
|
|
5971
|
-
|
|
6264
|
+
"""A SELECT or WITH statement, possibly after comments or parentheses."""
|
|
6265
|
+
head = text
|
|
6266
|
+
for _ in range(100):
|
|
6267
|
+
head = head.lstrip().lstrip("(").lstrip()
|
|
6268
|
+
if head.startswith("--"):
|
|
6269
|
+
newline = head.find("\n")
|
|
6270
|
+
if newline < 0:
|
|
6271
|
+
return False
|
|
6272
|
+
head = head[newline + 1 :]
|
|
6273
|
+
elif head.startswith("/*"):
|
|
6274
|
+
end = head.find("*/")
|
|
6275
|
+
if end < 0:
|
|
6276
|
+
return False
|
|
6277
|
+
head = head[end + 2 :]
|
|
6278
|
+
else:
|
|
6279
|
+
break
|
|
6280
|
+
return re.match(r"(select|with)\s", head, re.IGNORECASE) is not None
|
|
5972
6281
|
|
|
5973
6282
|
def _promote(self, name: str, value: _Value, context: _Context) -> _Value:
|
|
5974
6283
|
"""Bind an unresolved call result as an inferred-origin DataFrame when used as one."""
|
|
@@ -5994,7 +6303,7 @@ class _Analyzer:
|
|
|
5994
6303
|
"the extraction explicit.",
|
|
5995
6304
|
)
|
|
5996
6305
|
if opaque.query is not None:
|
|
5997
|
-
item = Extraction("sql", opaque.callee, query=opaque.query)
|
|
6306
|
+
item = Extraction("sql", opaque.callee, query=opaque.query, dialect=opaque.dialect)
|
|
5998
6307
|
self._note(
|
|
5999
6308
|
context,
|
|
6000
6309
|
node,
|
|
@@ -7933,7 +8242,14 @@ def analyze_project(
|
|
|
7933
8242
|
}
|
|
7934
8243
|
if len({item.callable for item in source_contracts}) != len(source_contracts):
|
|
7935
8244
|
raise ValueError("Source contracts must have unique callable identities")
|
|
7936
|
-
analyzer = _Analyzer(
|
|
8245
|
+
analyzer = _Analyzer(
|
|
8246
|
+
modules,
|
|
8247
|
+
ambiguous,
|
|
8248
|
+
source_contracts,
|
|
8249
|
+
source_schemas,
|
|
8250
|
+
settings.observers,
|
|
8251
|
+
settings.sql_dialect,
|
|
8252
|
+
)
|
|
7937
8253
|
analyzer.run()
|
|
7938
8254
|
for schema in source_schemas:
|
|
7939
8255
|
if schema not in analyzer.used_source_schemas:
|