sqlleaf 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. sqlleaf/__init__.py +4 -0
  2. sqlleaf/dialects/__init__.py +0 -0
  3. sqlleaf/dialects/plpgsql/__init__.py +0 -0
  4. sqlleaf/dialects/plpgsql/p_classes.py +275 -0
  5. sqlleaf/dialects/plpgsql/p_dialect.py +148 -0
  6. sqlleaf/dialects/plpgsql/p_generator.py +538 -0
  7. sqlleaf/dialects/plpgsql/p_parser.py +1244 -0
  8. sqlleaf/dialects/plpgsql/p_tokenizer.py +41 -0
  9. sqlleaf/exception.py +67 -0
  10. sqlleaf/lineage.py +326 -0
  11. sqlleaf/mappings.py +376 -0
  12. sqlleaf/models/__init__.py +0 -0
  13. sqlleaf/models/context.py +153 -0
  14. sqlleaf/models/hook.py +9 -0
  15. sqlleaf/models/node/__init__.py +27 -0
  16. sqlleaf/models/node/base.py +148 -0
  17. sqlleaf/models/node/column.py +345 -0
  18. sqlleaf/models/node/dynamodb.py +39 -0
  19. sqlleaf/models/node/function.py +33 -0
  20. sqlleaf/models/node/interval.py +21 -0
  21. sqlleaf/models/node/json.py +57 -0
  22. sqlleaf/models/node/literal.py +17 -0
  23. sqlleaf/models/node/null.py +14 -0
  24. sqlleaf/models/node/pivot.py +34 -0
  25. sqlleaf/models/node/program.py +27 -0
  26. sqlleaf/models/node/sequence.py +18 -0
  27. sqlleaf/models/node/star.py +11 -0
  28. sqlleaf/models/node/stream.py +14 -0
  29. sqlleaf/models/node/user_defined_function.py +33 -0
  30. sqlleaf/models/node/var.py +7 -0
  31. sqlleaf/models/node/variable.py +8 -0
  32. sqlleaf/models/node/window.py +22 -0
  33. sqlleaf/models/query/__init__.py +33 -0
  34. sqlleaf/models/query/base.py +248 -0
  35. sqlleaf/models/query/call.py +111 -0
  36. sqlleaf/models/query/copy.py +144 -0
  37. sqlleaf/models/query/ctas.py +61 -0
  38. sqlleaf/models/query/database.py +48 -0
  39. sqlleaf/models/query/delete.py +26 -0
  40. sqlleaf/models/query/execute.py +66 -0
  41. sqlleaf/models/query/holder.py +49 -0
  42. sqlleaf/models/query/insert.py +69 -0
  43. sqlleaf/models/query/merge.py +29 -0
  44. sqlleaf/models/query/multitable_insert.py +42 -0
  45. sqlleaf/models/query/prepare.py +61 -0
  46. sqlleaf/models/query/procedure.py +72 -0
  47. sqlleaf/models/query/put.py +38 -0
  48. sqlleaf/models/query/replace.py +32 -0
  49. sqlleaf/models/query/schema.py +47 -0
  50. sqlleaf/models/query/select.py +27 -0
  51. sqlleaf/models/query/sequence.py +33 -0
  52. sqlleaf/models/query/set.py +74 -0
  53. sqlleaf/models/query/stage.py +78 -0
  54. sqlleaf/models/query/table.py +220 -0
  55. sqlleaf/models/query/trigger.py +42 -0
  56. sqlleaf/models/query/type.py +42 -0
  57. sqlleaf/models/query/unload.py +64 -0
  58. sqlleaf/models/query/update.py +45 -0
  59. sqlleaf/models/query/user_defined_function.py +297 -0
  60. sqlleaf/models/query/values.py +20 -0
  61. sqlleaf/models/query/view.py +41 -0
  62. sqlleaf/path.py +194 -0
  63. sqlleaf/processors/__init__.py +0 -0
  64. sqlleaf/processors/collector/__init__.py +0 -0
  65. sqlleaf/processors/collector/collector.py +827 -0
  66. sqlleaf/processors/collector/substitute.py +396 -0
  67. sqlleaf/processors/generator/__init__.py +0 -0
  68. sqlleaf/processors/generator/dialects/__init__.py +5 -0
  69. sqlleaf/processors/generator/dialects/athena.py +45 -0
  70. sqlleaf/processors/generator/dialects/base.py +572 -0
  71. sqlleaf/processors/generator/dialects/mysql.py +19 -0
  72. sqlleaf/processors/generator/dialects/postgres.py +182 -0
  73. sqlleaf/processors/generator/dialects/redshift.py +192 -0
  74. sqlleaf/processors/generator/dialects/snowflake.py +90 -0
  75. sqlleaf/processors/generator/generator.py +495 -0
  76. sqlleaf/processors/transformer/__init__.py +31 -0
  77. sqlleaf/processors/transformer/base.py +899 -0
  78. sqlleaf/processors/transformer/call.py +9 -0
  79. sqlleaf/processors/transformer/copy.py +63 -0
  80. sqlleaf/processors/transformer/ctas.py +39 -0
  81. sqlleaf/processors/transformer/delete.py +14 -0
  82. sqlleaf/processors/transformer/execute.py +9 -0
  83. sqlleaf/processors/transformer/expressions/__init__.py +11 -0
  84. sqlleaf/processors/transformer/expressions/functional_notation.py +22 -0
  85. sqlleaf/processors/transformer/expressions/row.py +451 -0
  86. sqlleaf/processors/transformer/expressions/values.py +318 -0
  87. sqlleaf/processors/transformer/insert.py +125 -0
  88. sqlleaf/processors/transformer/merge.py +14 -0
  89. sqlleaf/processors/transformer/multitable_insert.py +14 -0
  90. sqlleaf/processors/transformer/replace.py +23 -0
  91. sqlleaf/processors/transformer/transformer.py +141 -0
  92. sqlleaf/processors/transformer/udf.py +235 -0
  93. sqlleaf/processors/transformer/unload.py +30 -0
  94. sqlleaf/processors/transformer/update.py +174 -0
  95. sqlleaf/processors/transformer/values.py +10 -0
  96. sqlleaf/settings/__init__.py +19 -0
  97. sqlleaf/settings/athena.py +23 -0
  98. sqlleaf/settings/base.py +37 -0
  99. sqlleaf/settings/postgres.py +52 -0
  100. sqlleaf/system_tables/load.py +0 -0
  101. sqlleaf/typing.py +93 -0
  102. sqlleaf/util/__init__.py +34 -0
  103. sqlleaf/util/expression.py +282 -0
  104. sqlleaf/util/graph.py +116 -0
  105. sqlleaf/util/helpers.py +42 -0
  106. sqlleaf/util/iterators.py +37 -0
  107. sqlleaf-0.0.1.dist-info/METADATA +46 -0
  108. sqlleaf-0.0.1.dist-info/RECORD +111 -0
  109. sqlleaf-0.0.1.dist-info/WHEEL +5 -0
  110. sqlleaf-0.0.1.dist-info/licenses/LICENSE +21 -0
  111. sqlleaf-0.0.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,34 @@
1
+ from sqlleaf.util.expression import (
2
+ calculate_function_name as calculate_function_name,
3
+ column_def_to_column as column_def_to_column,
4
+ copy_expression as copy_expression,
5
+ find_property as find_property,
6
+ get_column_constraint_expression as get_column_constraint_expression,
7
+ get_function_args as get_function_args,
8
+ get_location_property as get_location_property,
9
+ get_selected_column_names as get_selected_column_names,
10
+ get_table as get_table,
11
+ get_udf_name as get_udf_name,
12
+ is_row_function as is_row_function,
13
+ qualify_and_annotate as qualify_and_annotate,
14
+ rename_if_stage as rename_if_stage,
15
+ str_to_column_def as str_to_column_def,
16
+ unwrap_expression as unwrap_expression,
17
+ )
18
+ from sqlleaf.util.graph import (
19
+ find_edges_downward as find_edges_downward,
20
+ find_paths as find_paths,
21
+ get_cycles as get_cycles,
22
+ get_root_nodes as get_root_nodes,
23
+ )
24
+ from sqlleaf.util.helpers import (
25
+ chunks as chunks,
26
+ flatten as flatten,
27
+ long_sha256_hash as long_sha256_hash,
28
+ short_sha256_hash as short_sha256_hash,
29
+ unique as unique,
30
+ )
31
+ from sqlleaf.util.iterators import (
32
+ default_column_index_iterator as default_column_index_iterator,
33
+ iter_inner_statements as iter_inner_statements,
34
+ )
@@ -0,0 +1,282 @@
1
+ import typing as t
2
+
3
+ from sqlglot import exp
4
+ from sqlglot.optimizer.annotate_types import annotate_types
5
+ from sqlglot.optimizer.qualify import qualify
6
+
7
+ from sqlleaf import exception, mappings, util
8
+ from sqlleaf.typing import E, SourceExprType, TargetExprType
9
+
10
+
11
+ def unwrap_expression(expr: E) -> exp.Expr:
12
+ """
13
+ Extract the expression from underneath an Alias or a Paren.
14
+ """
15
+ ex = expr
16
+ while isinstance(ex, (exp.Alias, exp.Paren)):
17
+ ex = ex.unalias()
18
+ if isinstance(ex, (exp.Subquery,)):
19
+ break
20
+ ex = ex.unnest()
21
+ return ex
22
+
23
+
24
+ def copy_expression(expr: E) -> E:
25
+ """
26
+ Copy an expression.
27
+
28
+ Unlike sqlglot's copy() method, this preserves the expression's parents.
29
+ """
30
+ for i, ex in enumerate(expr.root().walk()):
31
+ if ex == expr:
32
+ copy_expr = expr.root().copy()
33
+ # Get the equivalent statement in the copy
34
+ for j, new_ex in enumerate(copy_expr.walk()):
35
+ if j == i:
36
+ return t.cast(E, new_ex)
37
+ return expr
38
+
39
+
40
+ def calculate_function_name(expr: exp.Expr, dialect: str) -> str:
41
+ """
42
+ Remove everything from the first '(' to the last ')' from a string.
43
+ We use this method because exp.Func.sql_name() includes the function's context in its name.
44
+ """
45
+ try:
46
+ # Get the name without its parameters
47
+ name = expr.__class__().sql(dialect=dialect)
48
+ except TypeError, AttributeError:
49
+ # Some classes can't be converted to SQL using this method (e.g. CONCAT() in Postgres)
50
+ name = expr.__class__().sql()
51
+ if not name:
52
+ name = expr.sql(dialect=dialect)
53
+
54
+ first_bracket = name.find("(")
55
+ if first_bracket == -1:
56
+ return name
57
+
58
+ last_bracket = name.rfind(")")
59
+ if last_bracket == -1:
60
+ return name
61
+
62
+ return name[:first_bracket] + name[last_bracket + 1 :]
63
+
64
+
65
+ def find_property(statement: exp.Create, child_object: TargetExprType, dialect: str) -> str:
66
+ """
67
+ Get the table/view's property (e.g. TEMPORARY, EXTERNAL, RECURSIVE)
68
+ """
69
+ if dialect == "redshift" and isinstance(child_object, exp.Table) and child_object.name.startswith("#"):
70
+ return "temporary"
71
+
72
+ properties = (exp.TemporaryProperty, exp.ExternalProperty, exp.MaterializedProperty)
73
+ prop = ""
74
+ if props := statement.args.get("properties"):
75
+ prop = str(props.find(properties) or "").lower()
76
+ return prop
77
+
78
+
79
+ def get_location_property(expr: exp.Create, dialect: str) -> str | None:
80
+ """
81
+ Get the LOCATION value from a CREATE statement.
82
+ """
83
+ location = None
84
+ if dialect in ["athena"]:
85
+ if props := expr.args["properties"]:
86
+ if location_expr := props.find(exp.LocationProperty):
87
+ location = location_expr.name
88
+ return location
89
+
90
+
91
+ def column_def_to_column(column_def: exp.ColumnDef, parent_table: t.Optional[exp.Table] = None) -> exp.Column:
92
+ """
93
+ Convert an exp.ColumnDef to an exp.Column
94
+ """
95
+ if parent_table:
96
+ table = parent_table
97
+ elif isinstance(column_def.parent, exp.Schema):
98
+ table: exp.Table = column_def.parent.this
99
+ else:
100
+ table = column_def.parent
101
+
102
+ col = exp.column(
103
+ column_def.name,
104
+ table=table.name if table else None,
105
+ db=table.db if table else None,
106
+ catalog=table.catalog if table else None,
107
+ )
108
+ col.type = column_def.kind
109
+ return col
110
+
111
+
112
+ def str_to_column_def(name: str) -> exp.ColumnDef:
113
+ """
114
+ Convert a string into a ColumnDef.
115
+ """
116
+ return exp.ColumnDef(this=exp.to_identifier(name), kind=exp.DType.UNKNOWN.into_expr())
117
+
118
+
119
+ def get_table(expr: exp.Expr) -> exp.Table:
120
+ table = expr.find(exp.Table)
121
+ if table is None:
122
+ raise exception.InvalidQueryError(message=f"Could not find an exp.Table in expression: {expr.sql()}")
123
+ return table
124
+
125
+
126
+ def get_udf_name(expr: exp.Anonymous) -> tuple[str, str]:
127
+ """
128
+ Get the schema and function name of a UDF (e.g. from SELECT my.func())
129
+ """
130
+ if isinstance(expr.parent, (exp.Dot,)):
131
+ schema = str(expr.parent.left.name)
132
+ function = str(expr.parent.right.name)
133
+ else:
134
+ # A function without a schema
135
+ schema = ""
136
+ function = expr.name
137
+
138
+ return schema, function
139
+
140
+
141
+ def get_function_args(expr: exp.Func) -> t.List[exp.Expr]:
142
+ """
143
+ Get all the argument expressions inside a function.
144
+ Example: "a, b, c" in "SELECT my.func(a,b,c)"
145
+ """
146
+ function_args = list(expr.args.values())
147
+ function_args = util.flatten(function_args)
148
+ exclude = (exp.TableAlias,) # SELECT FROM UNNEST() AS u
149
+ result = []
150
+ for arg in function_args:
151
+ if arg and isinstance(arg, exp.Expr) and not isinstance(arg, exclude):
152
+ if isinstance(arg, exp.Order):
153
+ # STRING_AGG(Order(Column)) -> Column
154
+ result.append(arg.this)
155
+ else:
156
+ result.append(arg)
157
+ return result
158
+
159
+
160
+ def rename_if_stage(source: SourceExprType, target: TargetExprType) -> None:
161
+ """
162
+ Normalize (uppercase) the name if we are a Snowflake stage.
163
+ sqlglot only normalizes columns - see comments in `sqlglot.optimizer.normalize_identifiers()`
164
+ """
165
+ if str(source).startswith("@"):
166
+ if not str(source).startswith('@"'):
167
+ if isinstance(source, exp.Var):
168
+ source.set("this", str(source).upper())
169
+ else:
170
+ source.this.set("this", str(source).upper())
171
+
172
+ elif str(target).startswith("@"):
173
+ if not str(target).startswith('@"'):
174
+ if isinstance(target, exp.Var):
175
+ target.set("this", str(target).upper())
176
+ else:
177
+ target.this.set("this", str(target).upper())
178
+
179
+
180
+ def get_selected_column_names(statement: exp.Expr) -> t.List[str]:
181
+ if isinstance(statement.expression, exp.Values):
182
+ return [s.name for s in statement.this.expressions]
183
+ return [s.alias_or_name for s in statement.selects]
184
+
185
+
186
+ def is_row_function(expr: exp.Expr) -> bool:
187
+ """
188
+ Returns True if the given expression is the ROW() function.
189
+ """
190
+ return isinstance(expr, exp.Anonymous) and expr.this.upper() == "ROW"
191
+
192
+
193
+ def qualify_and_annotate(
194
+ expr: exp.Expr, dialect: str, object_mapping: mappings.ObjectMapping, remove_added_aliases: bool = False
195
+ ) -> None:
196
+ stmt = qualify(
197
+ expr,
198
+ schema=object_mapping,
199
+ expand_stars=True,
200
+ expand_alias_refs=True,
201
+ qualify_columns=True,
202
+ infer_schema=False,
203
+ dialect=dialect,
204
+ isolate_tables=False,
205
+ validate_qualify_columns=False,
206
+ quote_identifiers=False,
207
+ )
208
+
209
+ # Rename the aliases automatically added by sqlglot
210
+ if remove_added_aliases:
211
+ if not isinstance(stmt.expression, exp.Values):
212
+ named_columns = stmt.args["this"].expressions
213
+ for i, ins in enumerate(named_columns):
214
+ # Overwrite the aliases because sqlglot may have added incorrect ones
215
+ if isinstance(ins, exp.ColumnDef):
216
+ ins = ins.this
217
+ stmt.selects[i] = stmt.selects[i].as_(ins)
218
+
219
+ annotate_types(stmt, dialect=dialect, schema=object_mapping)
220
+
221
+
222
+ def get_expression_for_column(column: exp.Column | int, expr: exp.Expr) -> tuple[exp.Expr, int]:
223
+ """
224
+ Get the expression that matches the given column name, along with its index (position) in the SELECT.
225
+ For example, given "SELECT 1 AS a, 2 AS b", column 'b' maps to expression 2.
226
+ """
227
+ if isinstance(column, int):
228
+ # The index of the query in "SELECT 1 UNION SELECT 2"
229
+ select = getattr(expr, "selects")[column]
230
+ idx = column
231
+ else:
232
+ if isinstance(expr, exp.Lateral):
233
+ selects = [(expr, 0)]
234
+ else:
235
+ # Common path
236
+ selects = [
237
+ (select, idx)
238
+ for idx, select in enumerate(getattr(expr, "selects"))
239
+ if select.alias_or_name == column.name
240
+ ]
241
+
242
+ if len(selects) > 1:
243
+ message = f"Column reference '{column}' is ambiguous ({len(selects)} possible options)"
244
+ raise exception.MappingError(message)
245
+
246
+ if selects:
247
+ select, idx = selects[0]
248
+ else:
249
+ select = expr
250
+ idx = -1
251
+ return select, idx
252
+
253
+
254
+ def get_column_index(column: exp.Column | int, expr: exp.Expr) -> int:
255
+ """
256
+ Return the positional index of a column within a SELECT list.
257
+ Raises if the column cannot be found.
258
+ """
259
+ index = (
260
+ column
261
+ if isinstance(column, int)
262
+ else next(
263
+ (i for i, sel in enumerate(getattr(expr, "selects")) if sel.alias_or_name == column.name),
264
+ -1, # a negative index should never be returned on success
265
+ )
266
+ )
267
+ if index == -1:
268
+ col_name = column if isinstance(column, int) else column.name
269
+ raise exception.MappingError(message=f"Could not find {col_name} in {expr}")
270
+ return index
271
+
272
+
273
+ def get_column_constraint_expression(expr: exp.ColumnDef) -> exp.ColumnConstraintKind | None:
274
+ """
275
+ Get the DEFAULT or GENERATED expression for this column, if it exists.
276
+ There is only one, but this
277
+ """
278
+ types = (exp.DefaultColumnConstraint, exp.ComputedColumnConstraint)
279
+ constraints = [
280
+ c.kind for c in expr.constraints if isinstance(c, exp.ColumnConstraint) and isinstance(c.kind, types)
281
+ ]
282
+ return t.cast(exp.ColumnConstraintKind, constraints[0]) if constraints else None
sqlleaf/util/graph.py ADDED
@@ -0,0 +1,116 @@
1
+ import logging
2
+ import typing as t
3
+
4
+ import networkx as nx
5
+
6
+ from sqlleaf import util
7
+
8
+ logger = logging.getLogger("sqlleaf")
9
+
10
+
11
+ # TODO: add a function called 'find_unique_paths' to prevent traversal of duplicate edges
12
+
13
+
14
+ def find_edges_downward(g: nx.MultiDiGraph, node: str, seen: t.Optional[t.Set[str]] = None, depth: int = 0):
15
+ """
16
+ Traverse the graph, returning any unseen edges.
17
+
18
+ Similar to find_edges_from_root(), except we return an unseen edge found at each hop,
19
+ rather than the entire path leading us there.
20
+ """
21
+ if seen is None:
22
+ seen = {node}
23
+
24
+ # Get direct descendants
25
+ desc = nx.descendants_at_distance(g, node, 1)
26
+
27
+ for n in desc:
28
+ if n not in seen:
29
+ # TODO: this could be a bug similar to the above comment in function
30
+ edges = g.get_edge_data(node, n)
31
+ for idx, data in edges.items():
32
+ hop = data["attrs"]
33
+ # Depth-first search?
34
+ yield depth, hop
35
+ yield from find_edges_downward(g, n, seen.union([n]), depth + 1)
36
+
37
+
38
+ def find_paths(g: nx.MultiDiGraph, start=0, path: t.Optional[t.List[int]] = None, seen: t.Optional[t.Set[int]] = None):
39
+ """
40
+ Find all the complete paths in a graph by traversing the descendants of a node until we find
41
+ a node without any descendants.
42
+ """
43
+ if path is None:
44
+ path = [start]
45
+ if seen is None:
46
+ seen = {start}
47
+
48
+ # Get direct descendants
49
+ desc = nx.descendants_at_distance(g, start, 1)
50
+ if not desc:
51
+ yield path
52
+ else:
53
+ for n in desc:
54
+ if n in seen:
55
+ yield path
56
+ else:
57
+ yield from find_paths(g, n, path + [n], seen.union([n]))
58
+
59
+
60
+ def get_root_nodes(graph: nx.MultiDiGraph) -> t.List[str]:
61
+ """
62
+ Get the root node of a graph. A root node has no parents.
63
+ """
64
+ selfloops = []
65
+
66
+ def remove_selfloop_edges(n1: str, n2: str, edge_key: int):
67
+ attrs = graph[n1][n2][edge_key]["attrs"]
68
+ if n1 == n2 and attrs:
69
+ if n1 not in selfloops:
70
+ selfloops.append(n1)
71
+ return False
72
+ return True
73
+
74
+ # Remove all the selfloop edges so that we can find the root node,
75
+ # and then add them back. (This is due to in/out_degree() inclduing them as edges)
76
+ view = nx.subgraph_view(graph, filter_edge=remove_selfloop_edges)
77
+ roots = [
78
+ n
79
+ for n in view.nodes
80
+ if
81
+ # A root node
82
+ (view.in_degree(n) == 0 and view.out_degree(n) > 0)
83
+ or
84
+ # A selfloop
85
+ (n in selfloops and view.degree(n) == 0)
86
+ ]
87
+ if selfloops:
88
+ logger.debug(f"Found selfloops: {selfloops}")
89
+ return roots
90
+
91
+
92
+ def get_cycles(graph: nx.MultiDiGraph):
93
+ """
94
+ Find all cycles in a graph.
95
+ """
96
+ errors = 0
97
+
98
+ for cycle in nx.simple_cycles(graph):
99
+ columns = [col for col in cycle if col.startswith("column")]
100
+
101
+ if len(util.unique(columns)) == 1:
102
+ # A valid cycle. This is a selfloop that passes through another node, e.g. a function.
103
+ first_column = columns[0]
104
+ idx = cycle.index(first_column)
105
+ new_cycle = cycle[idx:] + cycle[:idx] + [first_column]
106
+ logger.debug(f"Found valid cycle in graph: {new_cycle}")
107
+ cycle = new_cycle
108
+ else:
109
+ if len(columns) == 0:
110
+ logger.error(f"A cycle must contain 1 column node: {cycle}")
111
+ errors = 1
112
+ elif len(util.unique(columns)) > 1:
113
+ logger.error(f"A cycle cannot contain more than 1 column (found {len(util.unique(columns))}): {cycle}")
114
+ errors = 1
115
+
116
+ yield cycle, errors
@@ -0,0 +1,42 @@
1
+ import hashlib
2
+ import typing as t
3
+
4
+
5
+ def unique(sequence: t.List):
6
+ """
7
+ Return a list of unique elements in a list while preserving insertion order.
8
+ """
9
+ seen = set()
10
+ return [x for x in sequence if not (x in seen or seen.add(x))]
11
+
12
+
13
+ def flatten(lst: t.List):
14
+ """
15
+ Flatten a potentially nested list into a single list.
16
+ For example,
17
+ [a, 1, [b, c]]
18
+ returns
19
+ [a, 1, b, c]
20
+ """
21
+ result = []
22
+ for item in lst:
23
+ if isinstance(item, list):
24
+ result.extend(flatten(item))
25
+ else:
26
+ result.append(item)
27
+ return result
28
+
29
+
30
+ def chunks(lst, n):
31
+ """
32
+ Yield successive n-sized chunks from lst.
33
+ """
34
+ return [lst[i : i + n] for i in range(0, len(lst), n)]
35
+
36
+
37
+ def short_sha256_hash(text: str):
38
+ return hashlib.md5(text.encode()).hexdigest()[:16]
39
+
40
+
41
+ def long_sha256_hash(text: str):
42
+ return hashlib.md5(text.encode()).hexdigest()
@@ -0,0 +1,37 @@
1
+ import typing as t
2
+
3
+ import sqlglot
4
+ from sqlglot import exp
5
+
6
+
7
+ def default_column_index_iterator(dialect: str, elems: t.List[t.Any]) -> t.Generator[str]:
8
+ """
9
+ Generates default columns defined for a specific SQL dialect.
10
+ """
11
+ for i in range(len(elems)):
12
+ if dialect == "postgres":
13
+ yield f"column{i + 1}"
14
+ elif dialect == "mysql":
15
+ yield f"column_{i}"
16
+ else:
17
+ yield f"column{i + 1}"
18
+
19
+
20
+ def iter_inner_statements(stmt: exp.Expr, dialect: str, wrap: bool = False) -> t.List[exp.Expr]:
21
+ """
22
+ Iterate over the inner statements of a given expression.
23
+ """
24
+ if isinstance(stmt, (exp.Literal, exp.Heredoc)):
25
+ body_text = stmt.this.strip()
26
+ try:
27
+ return sqlglot.parse(body_text, read=dialect)
28
+ except Exception:
29
+ return []
30
+ if isinstance(stmt, exp.Block):
31
+ return stmt.expressions
32
+ if isinstance(stmt, exp.Return):
33
+ return [exp.select(stmt.this)]
34
+
35
+ if wrap:
36
+ return [exp.select(stmt)]
37
+ return [stmt]
@@ -0,0 +1,46 @@
1
+ Metadata-Version: 2.4
2
+ Name: sqlleaf
3
+ Version: 0.0.1
4
+ Summary: Trace your data
5
+ Author-email: zac <zac@sqlleaf.io>
6
+ Project-URL: Homepage, https://github.com/sqlleaf-io/sqlleaf
7
+ Requires-Python: >=3.14
8
+ Description-Content-Type: text/markdown
9
+ License-File: LICENSE
10
+ Requires-Dist: build>=1.6.1
11
+ Requires-Dist: networkx==3.6.1
12
+ Requires-Dist: sqlglot==30.17.0
13
+ Dynamic: license-file
14
+
15
+ # sqlleaf
16
+ ## Trace your data
17
+ sqlleaf is a tool to help you to understand how your data flows across your data systems by producing detailed, column-level lineage across any SQL query and dialect.
18
+
19
+ ## Example
20
+
21
+ ```sql
22
+ -- File: example.sql
23
+ CREATE TABLE users (name VARCHAR(50), birthday DATE);
24
+ CREATE TABLE logins (name VARCHAR(50));
25
+
26
+ INSERT INTO users (name, birthday)
27
+ SELECT LOWER(name) AS name, CURRENT_TIMESTAMP AS birthday
28
+ FROM logins;
29
+ ```
30
+
31
+ ```python
32
+ sql = open("example.sql").read()
33
+
34
+ import sqlleaf
35
+ lineage = sqlleaf.Lineage()
36
+ lineage.generate(sql=sql, dialect="postgres")
37
+ lineage.print_paths()
38
+ ```
39
+ Output:
40
+ ```
41
+ column[logins.name] -> function[LOWER] -> column[users.name]
42
+ function[CURRENT_TIMESTAMP] -> column[users.birthday]
43
+ ```
44
+
45
+ ## Development
46
+ This project is under heavy development; version 0.1 will be released soon.
@@ -0,0 +1,111 @@
1
+ sqlleaf/__init__.py,sha256=2WJBC8zD0ZbvtO-CQOaiJx1wggu7kMsXgSTl0dgVJtc,331
2
+ sqlleaf/exception.py,sha256=TfMyvn-2oUWCfn8BI0370hIcZTIR8bdJu3hMX_mZZO4,1254
3
+ sqlleaf/lineage.py,sha256=Z6WQ5FfegxRf3BxwAZVsKZXx9X71HpkTs-SkGJMIWZs,12099
4
+ sqlleaf/mappings.py,sha256=QxdxWzvMRxvk2qpRvnWYf0E9EvF2VRoI_RM88KBccak,14821
5
+ sqlleaf/path.py,sha256=C9JanxW1tx_o7L_PbUx145W-oKvWdHdQBokWvMNMxPg,6490
6
+ sqlleaf/typing.py,sha256=U5-CQ8jiaZf8USNitpYp3skJAGXe_blpzt_mDECFMw0,2082
7
+ sqlleaf/dialects/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ sqlleaf/dialects/plpgsql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
+ sqlleaf/dialects/plpgsql/p_classes.py,sha256=HYvrgPNX6LqJltVrnanMzGeXdUP3WzA6yVRXF6YRaVU,6033
10
+ sqlleaf/dialects/plpgsql/p_dialect.py,sha256=xYGbKCVE3Cz4biyyeCckAudYShfGQRy2Sio00sdS-5U,5705
11
+ sqlleaf/dialects/plpgsql/p_generator.py,sha256=d0QQicRu29zomTgpk_g5euAhL2xaZTm-0zaDrS2KRMQ,22659
12
+ sqlleaf/dialects/plpgsql/p_parser.py,sha256=VisP709VNnLn8edr8KKqBZfCeFPNuYt23o1fGVx-mZ8,47113
13
+ sqlleaf/dialects/plpgsql/p_tokenizer.py,sha256=TDKMAlVvbO2DRg3RstZtXoYXnhTBQobVRC4sYsXSOpM,918
14
+ sqlleaf/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
15
+ sqlleaf/models/context.py,sha256=u2a8-ZKy-225HmpDm_iJ1LsmSPZc04Csh_I7J7EW-7g,5298
16
+ sqlleaf/models/hook.py,sha256=X-pNZ7lU_pOVIChmIjo1vjPZqyDXa9oblI5T1I95gbs,219
17
+ sqlleaf/models/node/__init__.py,sha256=5AtY0A6e4cSjReYTVrREqZvBjMvwz0xZJ4kNAA37jGo,1669
18
+ sqlleaf/models/node/base.py,sha256=uh6IZZQBmTcS5DQOKYQQ3ism7IuHeQGhi_tg4vbrXHs,3979
19
+ sqlleaf/models/node/column.py,sha256=gXR1mJvIotLzneNXPZoZUp6BwNco7lUdLP_SS2paGkc,11450
20
+ sqlleaf/models/node/dynamodb.py,sha256=QzQ2JOiEjcdgevpZCIzpx-Z-xNzq22_XnJfWfiriMVU,951
21
+ sqlleaf/models/node/function.py,sha256=wnurklIdp3UXAKoH9l9yxZPDdgfxbqgNObzFyochXLg,978
22
+ sqlleaf/models/node/interval.py,sha256=xAQDGKZxMPjh4q93ICNyIGLESefbscgL5eG0th2bAVE,605
23
+ sqlleaf/models/node/json.py,sha256=Uaqe19bGF5rXbpq5QbfI1KXGObAMk6mvm_uKi-aJsK4,1707
24
+ sqlleaf/models/node/literal.py,sha256=cI_a5l8D7WhMdIaXlrlmFoxD9fOZz59ESriMJSY5A50,408
25
+ sqlleaf/models/node/null.py,sha256=TIxp3495_qiGPD5yPkhw9s1T7w_p5j9hWhLBu8DmsyE,288
26
+ sqlleaf/models/node/pivot.py,sha256=TVx8dpaGRGNnyb6yxci_re_PBG3cwf_FeZZV8NpOxNs,891
27
+ sqlleaf/models/node/program.py,sha256=6OuVJ7Lc7J580HPNQcBbuanWc62_YkfQ3-7xDKpqd1o,920
28
+ sqlleaf/models/node/sequence.py,sha256=Ur0yDteImYhhzphvKfP5RUswtlbEt4SZcJey-GBdWJg,565
29
+ sqlleaf/models/node/star.py,sha256=mAJZXHOAgCVgKjQz8Al5A0y6zRq-x9Y-epTQmSya8Xw,220
30
+ sqlleaf/models/node/stream.py,sha256=wYsIdh-Mz4cMZeWf4-ittUrj367ysejwBg1lbLAcZdg,410
31
+ sqlleaf/models/node/user_defined_function.py,sha256=2-zVKY55MCkMgCRUrLCS7gewTzlzJwIzJ2cC8DVcmZ0,898
32
+ sqlleaf/models/node/var.py,sha256=6kFy0bD10Db3X4XAW6mIdQoDyJDiN49gKCuttOTkpEs,133
33
+ sqlleaf/models/node/variable.py,sha256=DQ4Jvz6iieoB-m_BHURTJuEycGYLDi-Xa7mfRhOytGg,169
34
+ sqlleaf/models/node/window.py,sha256=Bn9D2Hq3VRZ1KWIGY6-Tr0C5EVkWWI1uIrPfAosB7o8,618
35
+ sqlleaf/models/query/__init__.py,sha256=lIFcoT5SHB3XklSEqJ0Jt6Ydn9x2Sb26ov4UJXOHnlE,2049
36
+ sqlleaf/models/query/base.py,sha256=YdiNCBu3Poq-9pS6wcbPasLUzjsx6JjRVbUikqQuRu0,8847
37
+ sqlleaf/models/query/call.py,sha256=M1FkapforjstT5GI1tyGj86WIKFtkSw5AL2arfAbSjA,3247
38
+ sqlleaf/models/query/copy.py,sha256=Jkop-b6SolAYxT9htcMxRj5sk3Oa40U5yvpOZkgomRo,5156
39
+ sqlleaf/models/query/ctas.py,sha256=pRB32qOFiNdmjiMkOcFqqX1TbKEcM8FaRNDqvTSyxNU,2125
40
+ sqlleaf/models/query/database.py,sha256=g6OSYc_AKvmt-wSUKcWCn9KoJsAmjgPlO5vcZTIYGsk,1229
41
+ sqlleaf/models/query/delete.py,sha256=jHyR6wWrWbpTqR79eMh3Q0layO1oDpppzZzuETrgrTg,795
42
+ sqlleaf/models/query/execute.py,sha256=Lzoo6aj_GnAW_IIkZJAyImaoK-5UluKpWSr5i8go0xU,1704
43
+ sqlleaf/models/query/holder.py,sha256=PGVMkOBzGZvYbW_nH005RGXfm3fu-FgUWu_qhQ_xrXg,1838
44
+ sqlleaf/models/query/insert.py,sha256=Ga5be9z3TWWjSFpsmHryu5LzsaAqKK0-N0o8KOAJRek,2455
45
+ sqlleaf/models/query/merge.py,sha256=d-Zasi677TuQeHwYq4qtyMUnuRZks_ZuAFfPX8-GM0c,820
46
+ sqlleaf/models/query/multitable_insert.py,sha256=1HBH2OQLQ8Y9hGSPm7qcxvLWsIIMQE8zWP8NqfRvXr8,1313
47
+ sqlleaf/models/query/prepare.py,sha256=72MVxpIIjasD4V7WnFCGFFZc0_Pn64aTphRvYfFbkrc,2306
48
+ sqlleaf/models/query/procedure.py,sha256=Rg2z2jzTE-vaafiX9eyulgH0aA1EqtQKV4mc7U-SQjg,2278
49
+ sqlleaf/models/query/put.py,sha256=_vC2KuZ4DC5j-u3oHXRFOgxC3VvQ7VpHNq4ZBYES978,1141
50
+ sqlleaf/models/query/replace.py,sha256=cS4u5ueEn9SYUi1p9mJ1L9S0MW-bwQwFnUcJ3xYjY6E,899
51
+ sqlleaf/models/query/schema.py,sha256=MZ-lPZ626uYuRr8YeeJGODJIKFTnuN9MHL84g8SRIFY,1218
52
+ sqlleaf/models/query/select.py,sha256=YFfHXOwG20ZoBghKKpdxI-wjecyFSTmkEqHqISCgjHk,867
53
+ sqlleaf/models/query/sequence.py,sha256=YW9TM8GOZrCxhuAV6qtcyTsvFaHxiL8ZDzAJohXW_6Q,891
54
+ sqlleaf/models/query/set.py,sha256=wgVfZkuuiOtsJSmVllg1M3xj_C2jIwmPBRCuNbWziTc,2937
55
+ sqlleaf/models/query/stage.py,sha256=nR9WXQ-XJLEqlfO5KmWL2xaeXLuNTOS6UQpfQX55yn8,2240
56
+ sqlleaf/models/query/table.py,sha256=mVHBeQL88TrhbXGCwURMun97FD_9TaszFc_6wIm7uU0,9225
57
+ sqlleaf/models/query/trigger.py,sha256=MgknCT2VLCqsOCsYo8BKU5XGGjHuio3fJE2zTd0_qeQ,1416
58
+ sqlleaf/models/query/type.py,sha256=tavtoqIcn5Q7kS7s_94D5t-l8B2b0p_HqKJp3p48Kds,1255
59
+ sqlleaf/models/query/unload.py,sha256=tsdmlW_G09WpaN6SP89Uk5IhVV_vlm6z3ed0FRc8hBk,2624
60
+ sqlleaf/models/query/update.py,sha256=-RmuhzDEnKB_ulZs4_D_ef-IEWf0mW7fU6Y5cC4og6E,1365
61
+ sqlleaf/models/query/user_defined_function.py,sha256=nQ5gx9poj2IUabi5QePMQ0hd8JxzIFkiB8TDqGzj-o0,11506
62
+ sqlleaf/models/query/values.py,sha256=O-j8-EXegtZAUOKAiuCFyFOUaM9tXIJAv9R1kNGg7Bc,684
63
+ sqlleaf/models/query/view.py,sha256=dUdXwgX5AqhcknyNuxzfnJshm2d5s80M02LvpuX5RKM,1254
64
+ sqlleaf/processors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
65
+ sqlleaf/processors/collector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
66
+ sqlleaf/processors/collector/collector.py,sha256=yu2KjOb6bET4NRo7zWgpcnbCs72e7_FYI22tZpkIWgU,31049
67
+ sqlleaf/processors/collector/substitute.py,sha256=A59Oz3pm9ZfZ701-z9HcRILKJkioMKWlZZEJzIELp_g,14471
68
+ sqlleaf/processors/generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
69
+ sqlleaf/processors/generator/generator.py,sha256=gkKv3WIGIcfqL_KaIncofjuqnW3EeKvvb4deIuZ_6fg,18225
70
+ sqlleaf/processors/generator/dialects/__init__.py,sha256=D_odD7gs7EvS00eTX7JInqqgKRKyxWi_hKAfs1C4Vl0,478
71
+ sqlleaf/processors/generator/dialects/athena.py,sha256=UWaJwCWQ-KmKbH3dfPgCGpWP5TtvoMzKxJ_ieG3xxn4,1474
72
+ sqlleaf/processors/generator/dialects/base.py,sha256=fQrhHYUtItxt0UH9yA43jf8ugOsGuIlNgdRGcIvtzoU,22587
73
+ sqlleaf/processors/generator/dialects/mysql.py,sha256=Bqzr3GAUbX1C6gex4HshVB2e8NbJuj3YDtyRbA1KYVA,588
74
+ sqlleaf/processors/generator/dialects/postgres.py,sha256=E7J8F1ANWCwLpvsuZXDrbF_olb5TZ_sQyE704EAxOEU,7342
75
+ sqlleaf/processors/generator/dialects/redshift.py,sha256=J_XUBDpHiJqU0sbNlwGBfs8jGcBwhJk4-KdAe06b_Pg,7471
76
+ sqlleaf/processors/generator/dialects/snowflake.py,sha256=fGnppTapcFUDGNTMXCgAJzQhSPrnqxFpS0HVZqsmHZk,3127
77
+ sqlleaf/processors/transformer/__init__.py,sha256=EDt2LnAHXTx8wsTRUeZyY8rOxdndM0jC5nNyJFtqp_k,1302
78
+ sqlleaf/processors/transformer/base.py,sha256=n6NW3YXE1nMhKo9XKw6n4IU5txllkQ1c5lsZcX-qVE0,37421
79
+ sqlleaf/processors/transformer/call.py,sha256=jDr9z6-uQSP7piKpoVEAesns_wwcBGNpN4-UN5O37us,164
80
+ sqlleaf/processors/transformer/copy.py,sha256=a8EYrdtynC6FDPKkb3LsteJdzHQagTM6hh7Q1CDfUN0,2230
81
+ sqlleaf/processors/transformer/ctas.py,sha256=XgnDLTZQ2HAq19yVDrYPyE8vvtHAh003gQ3v44emz28,1603
82
+ sqlleaf/processors/transformer/delete.py,sha256=UXqVXygyxsei1liHWSMUht4KTQWNPxCMy_WyejPXZ2Q,375
83
+ sqlleaf/processors/transformer/execute.py,sha256=0-4fzmvDhqU98FGsWl62T9-pRGzVN-Y0MYA5Vio66NE,167
84
+ sqlleaf/processors/transformer/insert.py,sha256=JBauC6nU0rM4S3YZrGavazadE8apgCtbQ6iFYl1W_no,4644
85
+ sqlleaf/processors/transformer/merge.py,sha256=6kE_oiqIvAoy1C7FOx0s08FSJB9mYaDUttmbV7QyF0w,369
86
+ sqlleaf/processors/transformer/multitable_insert.py,sha256=mNwvBid_73BjLuyvHv4hdgR6D7Bo3vvTjBOo21H8tFY,413
87
+ sqlleaf/processors/transformer/replace.py,sha256=3Sd4rHPHZZULNh-Rv7PFKRy5VjMSfDnY2G1cI_5QqzQ,972
88
+ sqlleaf/processors/transformer/transformer.py,sha256=YtDHRy_MzoBY2pBDlfwGkWB2Mig6D6-66rveF2QAttI,5068
89
+ sqlleaf/processors/transformer/udf.py,sha256=wvXmCZxkcrAWkP0fhUfgYFz7wh3PjcQtwCSV0UU2Fdc,8820
90
+ sqlleaf/processors/transformer/unload.py,sha256=bc-_XQo-o9LV8sxys2fWqCzQAWVRJFIY0ZP1YatizaI,960
91
+ sqlleaf/processors/transformer/update.py,sha256=BZW-60IqVJvh_6XAoHiZs_W0cEB9R4Vp6tfXkaFodXw,7935
92
+ sqlleaf/processors/transformer/values.py,sha256=rBOaHiD3wu5skDTdYQbiOcrZrxq-LZjNYA-j1N3ytuI,362
93
+ sqlleaf/processors/transformer/expressions/__init__.py,sha256=WzhAQKYYJyJI00btAykFBLqLuqCSZABYsuG9a0SSjfs,508
94
+ sqlleaf/processors/transformer/expressions/functional_notation.py,sha256=r4bweR3kfhzwUmu3BOcbk4gq6uz42-8wKuQQrLeSdBU,809
95
+ sqlleaf/processors/transformer/expressions/row.py,sha256=6_ldPcjoeYm8OjPZPz2obO1t7ZPl0_Mibops6xhIVIg,16621
96
+ sqlleaf/processors/transformer/expressions/values.py,sha256=x6rDdoyudPck4zZLzwjl1U0yDVC0_PoQjP5LDQNmhSQ,12297
97
+ sqlleaf/settings/__init__.py,sha256=7hd3p03Wwi4CzKdwrpiP9bKYD4tANseWZzxp-e5bKk4,657
98
+ sqlleaf/settings/athena.py,sha256=z9kW3Ai0-HZBb8Z4D8XaJJa4EzprArwScT2W09mqBVY,616
99
+ sqlleaf/settings/base.py,sha256=ygwsWpUjbjW8HAKz9dG2p5NrQVQRVxYB0Ot3rMr7Qbw,985
100
+ sqlleaf/settings/postgres.py,sha256=W5PG6K_yNLLW-jNZ5Q2dIpeC42J0PPaAr5SafGiHFuM,1749
101
+ sqlleaf/system_tables/load.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
102
+ sqlleaf/util/__init__.py,sha256=Vw296gA0C7-0NicFrCE2gYJIXkug9lYFRZrcE9bbG78,1263
103
+ sqlleaf/util/expression.py,sha256=3iyE7OPXhnKsV1WxhrOMLVhiHbOZq8-a6KHeprgooNQ,9415
104
+ sqlleaf/util/graph.py,sha256=CdfMYzEZ1eAa3kRhrD7UBB0t_Szeoaap4x0IFu70Daw,3669
105
+ sqlleaf/util/helpers.py,sha256=44elo1HXDFlxcLNdAS0nD7xXAQgDkVFKvPZi4MJC06s,918
106
+ sqlleaf/util/iterators.py,sha256=NSPtlUmFA7pJ0HsBZtR8tpmzkfJ2OObGr9TbJVoWB2g,1045
107
+ sqlleaf-0.0.1.dist-info/licenses/LICENSE,sha256=z6iHh46VCvulJEPA9ZyH9v31vKaln0ZbwiUNicP6d1U,1071
108
+ sqlleaf-0.0.1.dist-info/METADATA,sha256=DXvopvClo7fEnATMF5Zx83GydfCf0yFxUqE00CR1GTM,1197
109
+ sqlleaf-0.0.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
110
+ sqlleaf-0.0.1.dist-info/top_level.txt,sha256=dokK2cq3jbXlILUgh6ymk5kGkJj7FqvTSBLGEzSRpjQ,8
111
+ sqlleaf-0.0.1.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+