sqlleaf 0.0.1__tar.gz → 0.0.2__tar.gz

sqlleaf-0.0.2/PKG-INFO

@@ -0,0 +1,229 @@
+Metadata-Version: 2.4
+Name: sqlleaf
+Version: 0.0.2
+Summary: Trace your data
+Author-email: zac <zac@sqlleaf.io>
+Project-URL: Homepage, https://github.com/sqlleaf-io/sqlleaf
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: networkx==3.6.1
+Requires-Dist: sqlglot==30.22.0
+Dynamic: license-file
+
+# sqlleaf
+## Trace your data
+sqlleaf is a tool to help you to understand how your data flows across your data systems by producing detailed, column-level lineage across any SQL query and dialect.
+
+It extends beyond traditional lineage tools by mapping relationships across all kinds of expressions, such as columns, functions, literals and variables.
+
+## Example
+Add some SQL queries to a file:
+```sql
+-- File: example.sql
+CREATE TABLE users (name VARCHAR(50), birthday DATE);
+CREATE TABLE logins (name VARCHAR(50));
+
+INSERT INTO users (name, birthday)
+SELECT LOWER(name) AS name, CURRENT_TIMESTAMP AS birthday
+FROM logins;
+```
+Add some Python code:
+```python
+sql = open("example.sql").read()
+
+import sqlleaf
+lineage = sqlleaf.Lineage()
+lineage.generate(sql=sql, dialect="postgres")
+lineage.print_paths()
+```
+View the lineage printed to the console:
+```
+column[logins.name] -> function[LOWER] -> column[users.name]
+function[CURRENT_TIMESTAMP] -> column[users.birthday]
+```
+
+## Applications
+sqlleaf allows you to build a wide range of new applications on top of SQL databases, such as:
+- data tracing and tagging systems, such as personally identifiable information tracking
+- lineage-based SQL validators
+- auditing systems and historical analysis
+- identification of unused columns and tables
+- support for database migration correctness
+- impact analysis of changes to queries
+
+## Features
+### Types of queries
+sqlleaf traces lineage across a wide range of DML queries, such as `INSERT`, `UPDATE`, `MERGE`, `CREATE TABLE AS`, `COPY`, `UNLOAD`, CTEs, and many more.
+
+It parses over 30 dialects, extending functionality from the underlying SQL transpiler `sqlglot`.
+
+Here are some basic queries as examples:
+```sql
+-- Basic CTAS
+CREATE TABLE dst AS SELECT * FROM src;
+
+-- Basic UPDATE
+UPDATE dst t SET first_name = 'john', age = s.age FROM src s;
+
+-- An INSERT with a CTE (Common Table Expression)
+WITH my_cte AS (SELECT 1 AS num) INSERT INTO dst (age) SELECT c.num FROM my_cte c;
+```
+
+### Support for dynamic SQL inside PL/pgSQL
+You can trace lineage across variables and complex, dynamically constructed statements. This works by analyzing the procedure's inner statements, substituting any arguments and variables, and transforming the statements into the desired queries.
+
+For example, given a query to calculate the row count of an arbitrary table:
+```sql
+-- Define a PL/pgSQL procedure in Postgres
+CREATE PROCEDURE get_maximum_id(schema_name TEXT, table_name TEXT)
+AS $$
+BEGIN
+    EXECUTE format('INSERT INTO target (my_value) SELECT MAX(id) FROM %I.%I', schema_name, table_name);
+END;
+$$ LANGUAGE plpgsql;
+
+-- Call it
+CALL get_maximum_id('public', 'users');
+```
+sqlleaf extracts and transforms this into the statement:
+```sql
+INSERT INTO target (my_value) SELECT MAX(id) FROM public.users;
+```
+and then generates its lineage:
+```
+column[public.users.id] -> function[MAX] -> column[target.my_value]
+```
+
+### User-defined functions, procedures and types
+Given a user-defined function or a procedure, sqlleaf will extract the statement's inner queries so that they can be referenced by the calling query:
+```sql
+-- Create the function
+CREATE FUNCTION hello() RETURNS TEXT AS $$
+    SELECT 'Hello';
+$$ LANGUAGE SQL;
+
+-- Insert its value into a new table
+CREATE TABLE dst (my_name VARCHAR);
+INSERT INTO dst (my_name) SELECT hello();
+```
+This will perform inline transformations to produce a new query:
+```sql
+INSERT INTO dst (my_name) SELECT (SELECT 'hello') AS name;
+```
+with lineage:
+`literal['Hello'] -> column[dst.name]`
+
+### Pivot, Unpivot
+Complex transformations such as pivots, unpivots, laterals and table functions are supported.
+
+```sql
+-- UNPIVOT example
+INSERT INTO dst
+SELECT name, amount
+FROM src
+UNPIVOT (
+  amount FOR name IN (john_total AS 'john')
+);
+```
+produces lineage: \
+`literal["john"] -> unpivot[] -> column[target.name]` \
+`column[source.john_total] -> unpivot[] -> column[target.amount]`
+
+## Customization
+You can bring your own logic for processing SQL expressions and their nodes. 
+
+For example, let's say you only want to include certain functions inside the lineage graph, such as `SUM()`. You would do this by defining a *user-defined hook* and that excludes everything except `SUM()`:
+```python
+import sqlleaf
+lineage = sqlleaf.Lineage()
+
+from sqlleaf.lineage import Hook
+from sqlleaf.models.node import FunctionNode
+
+# Register a hook that only collects certain functions
+hook = Hook(name="only_sum", kind=FunctionNode, func=lambda n: n.name == "SUM")
+lineage.register_hooks(hooks=[hook])
+```
+When you run `lineage.generate()`, the only functions that are included as nodes are those named `SUM`.
+
+## Searching the lineage
+To map the lineage into a graph, sqlleaf is built on top of the popular library `networkx`.
+If you need to search the graph, you can use any of network'x graph traversal functions.
+There are currently no dedicated search APIs for sqlleaf. These will be developed in an upcoming major release.
+
+There are various functions for printing the lineage: `print_paths()`, `print_tree()`, and `print_json()`.
+
+
+## Examples
+Below are examples of SQL queries and their corresponding lineages.
+
+#### 1. Simple INSERT
+```sql
+INSERT INTO dst (name) VALUES (3);
+```
+`literal[3] -> column[dst.name]`
+
+#### 2. JSONPath selectors
+```sql
+INSERT INTO dst SELECT jsonblob ->> 'fruits' -> 'apple' AS name FROM src;
+```
+`column[src.name] → jsonpath[.fruits.apple] → column[dst.name]`
+
+#### 3. Using variables via SET
+```snowflake
+SET b = 'src';
+CREATE TABLE dst AS SELECT col1 FROM IDENTIFIER($b);
+```
+`column[SRC.COL1] → column[DST.COL1]`
+
+#### 4. COPY queries
+```sql
+COPY dst TO STDOUT;
+```
+`column[dst.col1] → stream[stdout]` \
+`column[dst.col2] → stream[stdout] `
+
+```sql
+COPY dst TO PROGRAM 'gzip > /tmp/data.csv.gz';
+```
+`column[dst.col1] → program[gzip]` \
+`column[dst.col2] → program[gzip] `
+
+#### 5. UNLOAD query
+```redshift
+UNLOAD ('SELECT name FROM src') TO 's3://dst-bucket/name-prefix' IAM_ROLE '...';
+```
+`column[src.name] → column[name path=s3://dst-bucket/name-prefix]`
+
+#### 6. Sequences
+```sql
+CREATE SEQUENCE my_serial START 101;
+INSERT INTO dst (age) SELECT nextval('my_serial') as age;
+```
+`sequence[my_serial] → column[dst.age]`
+
+#### 7. An UPDATE with inner CTEs containing UPDATEs that return their old and new columns
+```sql
+WITH first_cte AS (
+    UPDATE fruit.raw
+    SET name = 'pear'
+    RETURNING age, old.age as old_age, new.age as new_age
+),
+second_cte AS (
+    UPDATE fruit.raw AS r
+    SET name = 'tomato'
+    RETURNING *, OLD.*, NEW.*
+)
+UPDATE fruit.processed
+SET age = first_cte.age
+FROM first_cte;
+```
+`column[fruit.raw.age] -> column[first_cte.age] -> column[fruit.processed.age]` \
+`literal["pear"] -> column[fruit.raw.name]` \
+`literal["tomato"] -> column[fruit.raw.name`
+
+## Limitations
+- Many of the dialects are incomplete. Postgres and PL/pgSQL have been mapped out extensively, but most of the dialects do not have their own specific features parsed into graphs yet. Dialects that do have features are Athena, Redshift, MySQL and Snowflake; the others have not been implemented at all yet.
+
+- sqlleaf does not support data deletion. Statements like `DELETE`, `TRUNCATE` or `DROP TABLE` currently have no effect. They will in the future.

sqlleaf-0.0.2/README.md

@@ -0,0 +1,216 @@
+# sqlleaf
+## Trace your data
+sqlleaf is a tool to help you to understand how your data flows across your data systems by producing detailed, column-level lineage across any SQL query and dialect.
+
+It extends beyond traditional lineage tools by mapping relationships across all kinds of expressions, such as columns, functions, literals and variables.
+
+## Example
+Add some SQL queries to a file:
+```sql
+-- File: example.sql
+CREATE TABLE users (name VARCHAR(50), birthday DATE);
+CREATE TABLE logins (name VARCHAR(50));
+
+INSERT INTO users (name, birthday)
+SELECT LOWER(name) AS name, CURRENT_TIMESTAMP AS birthday
+FROM logins;
+```
+Add some Python code:
+```python
+sql = open("example.sql").read()
+
+import sqlleaf
+lineage = sqlleaf.Lineage()
+lineage.generate(sql=sql, dialect="postgres")
+lineage.print_paths()
+```
+View the lineage printed to the console:
+```
+column[logins.name] -> function[LOWER] -> column[users.name]
+function[CURRENT_TIMESTAMP] -> column[users.birthday]
+```
+
+## Applications
+sqlleaf allows you to build a wide range of new applications on top of SQL databases, such as:
+- data tracing and tagging systems, such as personally identifiable information tracking
+- lineage-based SQL validators
+- auditing systems and historical analysis
+- identification of unused columns and tables
+- support for database migration correctness
+- impact analysis of changes to queries
+
+## Features
+### Types of queries
+sqlleaf traces lineage across a wide range of DML queries, such as `INSERT`, `UPDATE`, `MERGE`, `CREATE TABLE AS`, `COPY`, `UNLOAD`, CTEs, and many more.
+
+It parses over 30 dialects, extending functionality from the underlying SQL transpiler `sqlglot`.
+
+Here are some basic queries as examples:
+```sql
+-- Basic CTAS
+CREATE TABLE dst AS SELECT * FROM src;
+
+-- Basic UPDATE
+UPDATE dst t SET first_name = 'john', age = s.age FROM src s;
+
+-- An INSERT with a CTE (Common Table Expression)
+WITH my_cte AS (SELECT 1 AS num) INSERT INTO dst (age) SELECT c.num FROM my_cte c;
+```
+
+### Support for dynamic SQL inside PL/pgSQL
+You can trace lineage across variables and complex, dynamically constructed statements. This works by analyzing the procedure's inner statements, substituting any arguments and variables, and transforming the statements into the desired queries.
+
+For example, given a query to calculate the row count of an arbitrary table:
+```sql
+-- Define a PL/pgSQL procedure in Postgres
+CREATE PROCEDURE get_maximum_id(schema_name TEXT, table_name TEXT)
+AS $$
+BEGIN
+    EXECUTE format('INSERT INTO target (my_value) SELECT MAX(id) FROM %I.%I', schema_name, table_name);
+END;
+$$ LANGUAGE plpgsql;
+
+-- Call it
+CALL get_maximum_id('public', 'users');
+```
+sqlleaf extracts and transforms this into the statement:
+```sql
+INSERT INTO target (my_value) SELECT MAX(id) FROM public.users;
+```
+and then generates its lineage:
+```
+column[public.users.id] -> function[MAX] -> column[target.my_value]
+```
+
+### User-defined functions, procedures and types
+Given a user-defined function or a procedure, sqlleaf will extract the statement's inner queries so that they can be referenced by the calling query:
+```sql
+-- Create the function
+CREATE FUNCTION hello() RETURNS TEXT AS $$
+    SELECT 'Hello';
+$$ LANGUAGE SQL;
+
+-- Insert its value into a new table
+CREATE TABLE dst (my_name VARCHAR);
+INSERT INTO dst (my_name) SELECT hello();
+```
+This will perform inline transformations to produce a new query:
+```sql
+INSERT INTO dst (my_name) SELECT (SELECT 'hello') AS name;
+```
+with lineage:
+`literal['Hello'] -> column[dst.name]`
+
+### Pivot, Unpivot
+Complex transformations such as pivots, unpivots, laterals and table functions are supported.
+
+```sql
+-- UNPIVOT example
+INSERT INTO dst
+SELECT name, amount
+FROM src
+UNPIVOT (
+  amount FOR name IN (john_total AS 'john')
+);
+```
+produces lineage: \
+`literal["john"] -> unpivot[] -> column[target.name]` \
+`column[source.john_total] -> unpivot[] -> column[target.amount]`
+
+## Customization
+You can bring your own logic for processing SQL expressions and their nodes. 
+
+For example, let's say you only want to include certain functions inside the lineage graph, such as `SUM()`. You would do this by defining a *user-defined hook* and that excludes everything except `SUM()`:
+```python
+import sqlleaf
+lineage = sqlleaf.Lineage()
+
+from sqlleaf.lineage import Hook
+from sqlleaf.models.node import FunctionNode
+
+# Register a hook that only collects certain functions
+hook = Hook(name="only_sum", kind=FunctionNode, func=lambda n: n.name == "SUM")
+lineage.register_hooks(hooks=[hook])
+```
+When you run `lineage.generate()`, the only functions that are included as nodes are those named `SUM`.
+
+## Searching the lineage
+To map the lineage into a graph, sqlleaf is built on top of the popular library `networkx`.
+If you need to search the graph, you can use any of network'x graph traversal functions.
+There are currently no dedicated search APIs for sqlleaf. These will be developed in an upcoming major release.
+
+There are various functions for printing the lineage: `print_paths()`, `print_tree()`, and `print_json()`.
+
+
+## Examples
+Below are examples of SQL queries and their corresponding lineages.
+
+#### 1. Simple INSERT
+```sql
+INSERT INTO dst (name) VALUES (3);
+```
+`literal[3] -> column[dst.name]`
+
+#### 2. JSONPath selectors
+```sql
+INSERT INTO dst SELECT jsonblob ->> 'fruits' -> 'apple' AS name FROM src;
+```
+`column[src.name] → jsonpath[.fruits.apple] → column[dst.name]`
+
+#### 3. Using variables via SET
+```snowflake
+SET b = 'src';
+CREATE TABLE dst AS SELECT col1 FROM IDENTIFIER($b);
+```
+`column[SRC.COL1] → column[DST.COL1]`
+
+#### 4. COPY queries
+```sql
+COPY dst TO STDOUT;
+```
+`column[dst.col1] → stream[stdout]` \
+`column[dst.col2] → stream[stdout] `
+
+```sql
+COPY dst TO PROGRAM 'gzip > /tmp/data.csv.gz';
+```
+`column[dst.col1] → program[gzip]` \
+`column[dst.col2] → program[gzip] `
+
+#### 5. UNLOAD query
+```redshift
+UNLOAD ('SELECT name FROM src') TO 's3://dst-bucket/name-prefix' IAM_ROLE '...';
+```
+`column[src.name] → column[name path=s3://dst-bucket/name-prefix]`
+
+#### 6. Sequences
+```sql
+CREATE SEQUENCE my_serial START 101;
+INSERT INTO dst (age) SELECT nextval('my_serial') as age;
+```
+`sequence[my_serial] → column[dst.age]`
+
+#### 7. An UPDATE with inner CTEs containing UPDATEs that return their old and new columns
+```sql
+WITH first_cte AS (
+    UPDATE fruit.raw
+    SET name = 'pear'
+    RETURNING age, old.age as old_age, new.age as new_age
+),
+second_cte AS (
+    UPDATE fruit.raw AS r
+    SET name = 'tomato'
+    RETURNING *, OLD.*, NEW.*
+)
+UPDATE fruit.processed
+SET age = first_cte.age
+FROM first_cte;
+```
+`column[fruit.raw.age] -> column[first_cte.age] -> column[fruit.processed.age]` \
+`literal["pear"] -> column[fruit.raw.name]` \
+`literal["tomato"] -> column[fruit.raw.name`
+
+## Limitations
+- Many of the dialects are incomplete. Postgres and PL/pgSQL have been mapped out extensively, but most of the dialects do not have their own specific features parsed into graphs yet. Dialects that do have features are Athena, Redshift, MySQL and Snowflake; the others have not been implemented at all yet.
+
+- sqlleaf does not support data deletion. Statements like `DELETE`, `TRUNCATE` or `DROP TABLE` currently have no effect. They will in the future.

{sqlleaf-0.0.1 → sqlleaf-0.0.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sqlleaf"
-version = "0.0.1"
+version = "0.0.2"
 authors = [
     {name = "zac", email = "zac@sqlleaf.io"}
 ]
@@ -8,9 +8,14 @@ description = "Trace your data"
 readme = "README.md"
 requires-python = ">=3.14"
 dependencies = [
-    "build>=1.6.1",
     "networkx==3.6.1",
-    "sqlglot==30.17.0",
+    "sqlglot==30.22.0",
+]
+
+[dependency-groups]
+test = [
+    "pytest>=9.1.1",
+    "pytest-xdist>=3.8.0",
 ]
 
 [project.urls]
@@ -45,7 +50,4 @@ line-ending = "auto"
 
 [tool.ruff.lint]
 select = ["F", "E", "W", "I"]
-ignore = ["E501"]
-
-[tool.vulture]
-min_confidence = 100
+ignore = ["E501", "E731"]

{sqlleaf-0.0.1 → sqlleaf-0.0.2}/sqlleaf/__init__.py

@@ -2,3 +2,4 @@ from sqlleaf.lineage import Lineage as Lineage
 from sqlleaf.processors.collector.collector import collect_queries as collect_queries
 from sqlleaf.processors.generator.generator import generate_lineage_for_query as generate_lineage_for_query
 from sqlleaf.processors.transformer.transformer import transform_query as transform_query
+from sqlleaf.dialects import PlPgSQL

sqlleaf-0.0.2/sqlleaf/dialects/__init__.py

@@ -0,0 +1 @@
+from sqlleaf.dialects.plpgsql.p_dialect import PlPgSQL

sqlleaf-0.0.2/sqlleaf/dialects/plpgsql/__init__.py

@@ -0,0 +1 @@
+from sqlleaf.dialects.plpgsql import p_classes as pgexp

{sqlleaf-0.0.1 → sqlleaf-0.0.2}/sqlleaf/dialects/plpgsql/p_classes.py

@@ -146,7 +146,7 @@ class PGGetDiagnostics(exp.Expression):
     arg_types = {"current": False, "stacked": False, "expressions": True}
 
 
-class PGOpenCursor(exp.Expression):
+class PGOpen(exp.Expression):
     """An OPEN cursor statement."""
     arg_types = {"this": True, "expression": False, "scroll": False, "expressions": False}
 

{sqlleaf-0.0.1 → sqlleaf-0.0.2}/sqlleaf/dialects/plpgsql/p_generator.py

@@ -25,7 +25,7 @@ class PlPgSQLGenerator(PostgresGenerator):
         PGContinue: lambda self, e: self.pgcontinue_sql(e),
         PGRaise: lambda self, e: self.pgraise_sql(e),
         PGSqlState: lambda self, e: self.pgsqlstate_sql(e),
-        PGOpenCursor: lambda self, e: self.pgopencursor_sql(e),
+        PGOpen: lambda self, e: self.pgopen_sql(e),
         PGFetch: lambda self, e: self.pgfetch_sql(e),
         PGMove: lambda self, e: self.pgmove_sql(e),
         PGClose: lambda self, e: self.pgclose_sql(e),
@@ -419,7 +419,7 @@ class PlPgSQLGenerator(PostgresGenerator):
             return f"{self.sql(dir_expr)} {prep} {self.sql(expression.this)}"
         return self.sql(expression.this)
 
-    def pgopencursor_sql(self, expression: PGOpenCursor) -> str:
+    def pgopen_sql(self, expression: PGOpen) -> str:
         # Unbound form: OPEN c [NO|SCROLL] FOR <query>
         if expression.args.get("expression") is not None:
             parts: list[str] = ["OPEN", self.sql(expression.this)]

{sqlleaf-0.0.1 → sqlleaf-0.0.2}/sqlleaf/dialects/plpgsql/p_parser.py

@@ -1053,7 +1053,7 @@ class PlPgSQLParser(PostgresParser):
 
         return args
 
-    def _parse_pgopen(self) -> PGOpenCursor:
+    def _parse_pgopen(self) -> PGOpen:
         # Cursor variable identifier
         cursor = self._parse_id_var()
 
@@ -1062,7 +1062,7 @@ class PlPgSQLParser(PostgresParser):
         if self._match(TokenType.L_PAREN, advance=False):
             # Parse cursor call arguments directly (positional, name := value, name => value)
             args = self._parse_cursor_call_args()
-            return self.expression(PGOpenCursor(this=cursor, expressions=args))
+            return self.expression(PGOpen(this=cursor, expressions=args))
 
         # Optional [[NO] SCROLL]
         scroll: bool | None = None
@@ -1075,7 +1075,7 @@ class PlPgSQLParser(PostgresParser):
         # If there's no FOR and no SCROLL/NO SCROLL, treat as bound cursor with zero args: OPEN c;
         if not self._match(TokenType.FOR):
             if scroll is None:
-                return self.expression(PGOpenCursor(this=cursor))
+                return self.expression(PGOpen(this=cursor))
             # If SCROLL/NO SCROLL was provided, FOR is required
             self._match_expect(TokenType.FOR)
 
@@ -1088,7 +1088,7 @@ class PlPgSQLParser(PostgresParser):
             if query is None:
                 self.raise_error("Expected query after OPEN ... FOR")
 
-        return self.expression(PGOpenCursor(this=cursor, expression=query, scroll=scroll))
+        return self.expression(PGOpen(this=cursor, expression=query, scroll=scroll))
 
     def _parse_pgfetch(self) -> PGFetch:
         # Parse optional direction, optional preposition (required when direction given), and cursor

{sqlleaf-0.0.1 → sqlleaf-0.0.2}/sqlleaf/exception.py

@@ -6,12 +6,12 @@ ERROR_LEVEL = "stop"
 logger = logging.getLogger("sqlleaf")
 
 
-def raise_error(clazz: type, message: str):
+def raise_error(cls: type, message: str):
     """
     Raise a custom exception based on the error level.
     """
     if ERROR_LEVEL == "stop":
-        raise clazz(message)
+        raise cls(message)
     elif ERROR_LEVEL == "continue":
         logger.error(message)