sql-metadata 2.19.0__tar.gz → 3.0.0__tar.gz

{sql_metadata-2.19.0 → sql_metadata-3.0.0}/PKG-INFO

@@ -1,21 +1,20 @@
 Metadata-Version: 2.4
 Name: sql_metadata
-Version: 2.19.0
-Summary: Uses tokenized query returned by python-sqlparse and generates query metadata
+Version: 3.0.0
+Summary: Uses sqlglot to parse SQL queries and extract metadata
 License: MIT
 License-File: LICENSE
 Author: Maciej Brencz
 Author-email: maciej.brencz@gmail.com
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.10,<4.0
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
-Requires-Dist: sqlparse (>=0.4.1,<0.6.0)
+Requires-Dist: sqlglot (>=30.0.3,<31.0.0)
 Project-URL: Homepage, https://github.com/macbre/sql-metadata
 Project-URL: Repository, https://github.com/macbre/sql-metadata
 Description-Content-Type: text/markdown
@@ -25,11 +24,11 @@ Description-Content-Type: text/markdown
 [![PyPI](https://img.shields.io/pypi/v/sql_metadata.svg)](https://pypi.python.org/pypi/sql_metadata)
 [![Tests](https://github.com/macbre/sql-metadata/actions/workflows/python-ci.yml/badge.svg)](https://github.com/macbre/sql-metadata/actions/workflows/python-ci.yml)
 [![Coverage Status](https://coveralls.io/repos/github/macbre/sql-metadata/badge.svg?branch=master&1)](https://coveralls.io/github/macbre/sql-metadata?branch=master)
-<a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![Maintenance](https://img.shields.io/badge/maintained%3F-yes-green.svg)](https://github.com/macbre/sql-metadata/graphs/commit-activity)
 [![Downloads](https://pepy.tech/badge/sql-metadata/month)](https://pepy.tech/project/sql-metadata)
 
-Uses tokenized query returned by [`python-sqlparse`](https://github.com/andialbrecht/sqlparse) and generates query metadata.
+Uses [`sqlglot`](https://github.com/tobymao/sqlglot) to parse SQL queries and extract metadata.
 
 **Extracts column names and tables** used by the query. 
 Automatically conduct **column alias resolution**, **sub queries aliases resolution** as well as **tables aliases resolving**.
@@ -80,7 +79,7 @@ parser = Parser("SELECT a.* FROM product_a.users AS a JOIN product_b.users AS b
 
 # note that aliases are auto-resolved
 parser.columns
-# ['product_a.*', 'product_a.users.ip_address', 'product_b.users.ip_address']
+# ['product_a.users.*', 'product_a.users.ip_address', 'product_b.users.ip_address']
 
 # note that you can also extract columns with their place in the query
 # which will return dict with lists divided into select, where, order_by, group_by, join, insert and update
@@ -116,6 +115,71 @@ assert parser.columns_dict == {'order_by': ['b', 'c', 'u'],
                                'select': ['a', 'b', 'c', 'u', 'd']}
 ```
 
+### Extracting output column names
+
+```python
+from sql_metadata import Parser
+
+# output_columns returns the ordered list of names that the SELECT would produce,
+# preserving aliases (unlike `columns`, which resolves aliases back to real columns)
+Parser("SELECT a, b AS c FROM t").output_columns
+# ['a', 'c']
+
+# works with function calls, window functions, computed aliases
+Parser("""SELECT
+    id,
+    UPPER(email) AS email_upper,
+    ROW_NUMBER() OVER (PARTITION BY country ORDER BY created_at) AS rn
+FROM users""").output_columns
+# ['id', 'email_upper', 'rn']
+
+# SELECT * stays as '*'
+Parser("SELECT * FROM t").output_columns
+# ['*']
+
+# non-SELECT queries return an empty list
+Parser("CREATE TABLE t (id INT)").output_columns
+# []
+```
+
+### Detecting query type
+
+```python
+from sql_metadata import Parser, QueryType
+
+Parser("SELECT * FROM foo").query_type
+# <QueryType.SELECT: 'SELECT'>
+
+# QueryType is a str-enum, so it compares equal to both strings and enum values
+Parser("INSERT INTO foo VALUES (1)").query_type == QueryType.INSERT   # True
+Parser("INSERT INTO foo VALUES (1)").query_type == "INSERT"           # True
+
+# REPLACE INTO is reported distinctly from INSERT
+Parser("REPLACE INTO foo VALUES (1)").query_type
+# <QueryType.REPLACE: 'REPLACE'>
+
+# Supported types: SELECT, INSERT, REPLACE, UPDATE, DELETE,
+# CREATE, ALTER, DROP, TRUNCATE, MERGE
+```
+
+### Handling invalid queries
+
+```python
+from sql_metadata import Parser, InvalidQueryDefinition
+
+# structurally invalid SQL raises `InvalidQueryDefinition` (a subclass of
+# `ValueError`, so existing `except ValueError` handlers keep working)
+try:
+    Parser("").query_type
+except InvalidQueryDefinition as exc:
+    print(exc)  # "Empty queries are not supported!"
+
+try:
+    Parser("THIS IS NOT SQL").query_type
+except InvalidQueryDefinition as exc:
+    print(exc)  # "Not supported query type!"
+```
+
 ### Extracting tables from query
 
 ```python
@@ -203,9 +267,10 @@ parser.with_names
 # ["database1.tableFromWith", "test"]
 
 # get definition of with queries
+# (sqlglot normalises keyword casing and spacing when rendering the body SQL)
 parser.with_queries
-# {"database1.tableFromWith": "SELECT aa.* FROM table3 as aa left join table4 on aa.col1=table4.col2"
-#  "test": "SELECT * from table3"}
+# {"database1.tableFromWith": "SELECT aa.* FROM table3 AS aa LEFT JOIN table4 ON aa.col1 = table4.col2",
+#  "test": "SELECT * FROM table3"}
 
 # note that names of with statements do not appear in tables
 parser.tables
@@ -227,9 +292,10 @@ ON a.task_id = b.task_id;
 )
 
 # get sub-queries dictionary
+# (sqlglot normalises keyword casing — implicit table aliases become explicit `AS`)
 parser.subqueries
-# {"a": "SELECT std.task_id FROM some_task_detail std WHERE std.STATUS = 1",
-#  "b": "SELECT st.task_id FROM some_task st WHERE task_type_id = 80"}
+# {"a": "SELECT std.task_id FROM some_task_detail AS std WHERE std.STATUS = 1",
+#  "b": "SELECT st.task_id FROM some_task AS st WHERE task_type_id = 80"}
 
 
 # get names/ aliases of sub-queries / derived tables
@@ -272,30 +338,23 @@ parser.comments
 
 See `test/test_normalization.py` file for more examples of a bit more complex queries.
 
-## Migrating from `sql_metadata` 1.x
-
-`sql_metadata.compat` module has been implemented to make the introduction of sql-metadata v2.0 smoother.
+## Migrating from `sql_metadata` 1.x / 2.x
 
-You can use it by simply changing the imports in your code from:
+The `sql_metadata.compat` module (previously provided for v1 → v2 migration) has been **removed in v3**.  Port your code to the class-based `Parser` API shown in the examples above:
 
-```python
-from sql_metadata import get_query_columns, get_query_tables
-```
-
-into:
-
-```python
-from sql_metadata.compat import get_query_columns, get_query_tables
-```
+| Old v1 helper                 | v3 replacement                            |
+|-------------------------------|-------------------------------------------|
+| `generalize_sql(sql)`         | `Parser(sql).generalize`                  |
+| `get_query_columns(sql)`      | `Parser(sql).columns`                     |
+| `get_query_tables(sql)`       | `Parser(sql).tables`                      |
+| `get_query_limit_and_offset(sql)` | `Parser(sql).limit_and_offset`        |
+| `get_query_tokens(sql)`       | `Parser(sql).tokens`                      |
+| `preprocess_query(sql)`       | `Parser(sql).query`                       |
 
-The following functions from the old API are available in the `sql_metadata.compat` module:
+For v2 → v3 users, the public `Parser` API is unchanged except:
 
-* `generalize_sql`
-* `get_query_columns` (since #131 columns aliases ARE NOT returned by this function)
-* `get_query_limit_and_offset`
-* `get_query_tables`
-* `get_query_tokens`
-* `preprocess_query`
+* The parsing engine is now [sqlglot](https://github.com/tobymao/sqlglot), which may normalise the *casing* and *spacing* of rendered CTE/subquery bodies (see the `with_queries` / `subqueries` examples above).
+* Malformed SQL now raises `InvalidQueryDefinition` (a `ValueError` subclass) instead of a plain `ValueError` — existing `except ValueError:` handlers continue to work.
 
 ## Authors and contributors
 

{sql_metadata-2.19.0 → sql_metadata-3.0.0}/README.md

@@ -3,11 +3,11 @@
 [![PyPI](https://img.shields.io/pypi/v/sql_metadata.svg)](https://pypi.python.org/pypi/sql_metadata)
 [![Tests](https://github.com/macbre/sql-metadata/actions/workflows/python-ci.yml/badge.svg)](https://github.com/macbre/sql-metadata/actions/workflows/python-ci.yml)
 [![Coverage Status](https://coveralls.io/repos/github/macbre/sql-metadata/badge.svg?branch=master&1)](https://coveralls.io/github/macbre/sql-metadata?branch=master)
-<a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![Maintenance](https://img.shields.io/badge/maintained%3F-yes-green.svg)](https://github.com/macbre/sql-metadata/graphs/commit-activity)
 [![Downloads](https://pepy.tech/badge/sql-metadata/month)](https://pepy.tech/project/sql-metadata)
 
-Uses tokenized query returned by [`python-sqlparse`](https://github.com/andialbrecht/sqlparse) and generates query metadata.
+Uses [`sqlglot`](https://github.com/tobymao/sqlglot) to parse SQL queries and extract metadata.
 
 **Extracts column names and tables** used by the query. 
 Automatically conduct **column alias resolution**, **sub queries aliases resolution** as well as **tables aliases resolving**.
@@ -58,7 +58,7 @@ parser = Parser("SELECT a.* FROM product_a.users AS a JOIN product_b.users AS b
 
 # note that aliases are auto-resolved
 parser.columns
-# ['product_a.*', 'product_a.users.ip_address', 'product_b.users.ip_address']
+# ['product_a.users.*', 'product_a.users.ip_address', 'product_b.users.ip_address']
 
 # note that you can also extract columns with their place in the query
 # which will return dict with lists divided into select, where, order_by, group_by, join, insert and update
@@ -94,6 +94,71 @@ assert parser.columns_dict == {'order_by': ['b', 'c', 'u'],
                                'select': ['a', 'b', 'c', 'u', 'd']}
 ```
 
+### Extracting output column names
+
+```python
+from sql_metadata import Parser
+
+# output_columns returns the ordered list of names that the SELECT would produce,
+# preserving aliases (unlike `columns`, which resolves aliases back to real columns)
+Parser("SELECT a, b AS c FROM t").output_columns
+# ['a', 'c']
+
+# works with function calls, window functions, computed aliases
+Parser("""SELECT
+    id,
+    UPPER(email) AS email_upper,
+    ROW_NUMBER() OVER (PARTITION BY country ORDER BY created_at) AS rn
+FROM users""").output_columns
+# ['id', 'email_upper', 'rn']
+
+# SELECT * stays as '*'
+Parser("SELECT * FROM t").output_columns
+# ['*']
+
+# non-SELECT queries return an empty list
+Parser("CREATE TABLE t (id INT)").output_columns
+# []
+```
+
+### Detecting query type
+
+```python
+from sql_metadata import Parser, QueryType
+
+Parser("SELECT * FROM foo").query_type
+# <QueryType.SELECT: 'SELECT'>
+
+# QueryType is a str-enum, so it compares equal to both strings and enum values
+Parser("INSERT INTO foo VALUES (1)").query_type == QueryType.INSERT   # True
+Parser("INSERT INTO foo VALUES (1)").query_type == "INSERT"           # True
+
+# REPLACE INTO is reported distinctly from INSERT
+Parser("REPLACE INTO foo VALUES (1)").query_type
+# <QueryType.REPLACE: 'REPLACE'>
+
+# Supported types: SELECT, INSERT, REPLACE, UPDATE, DELETE,
+# CREATE, ALTER, DROP, TRUNCATE, MERGE
+```
+
+### Handling invalid queries
+
+```python
+from sql_metadata import Parser, InvalidQueryDefinition
+
+# structurally invalid SQL raises `InvalidQueryDefinition` (a subclass of
+# `ValueError`, so existing `except ValueError` handlers keep working)
+try:
+    Parser("").query_type
+except InvalidQueryDefinition as exc:
+    print(exc)  # "Empty queries are not supported!"
+
+try:
+    Parser("THIS IS NOT SQL").query_type
+except InvalidQueryDefinition as exc:
+    print(exc)  # "Not supported query type!"
+```
+
 ### Extracting tables from query
 
 ```python
@@ -181,9 +246,10 @@ parser.with_names
 # ["database1.tableFromWith", "test"]
 
 # get definition of with queries
+# (sqlglot normalises keyword casing and spacing when rendering the body SQL)
 parser.with_queries
-# {"database1.tableFromWith": "SELECT aa.* FROM table3 as aa left join table4 on aa.col1=table4.col2"
-#  "test": "SELECT * from table3"}
+# {"database1.tableFromWith": "SELECT aa.* FROM table3 AS aa LEFT JOIN table4 ON aa.col1 = table4.col2",
+#  "test": "SELECT * FROM table3"}
 
 # note that names of with statements do not appear in tables
 parser.tables
@@ -205,9 +271,10 @@ ON a.task_id = b.task_id;
 )
 
 # get sub-queries dictionary
+# (sqlglot normalises keyword casing — implicit table aliases become explicit `AS`)
 parser.subqueries
-# {"a": "SELECT std.task_id FROM some_task_detail std WHERE std.STATUS = 1",
-#  "b": "SELECT st.task_id FROM some_task st WHERE task_type_id = 80"}
+# {"a": "SELECT std.task_id FROM some_task_detail AS std WHERE std.STATUS = 1",
+#  "b": "SELECT st.task_id FROM some_task AS st WHERE task_type_id = 80"}
 
 
 # get names/ aliases of sub-queries / derived tables
@@ -250,30 +317,23 @@ parser.comments
 
 See `test/test_normalization.py` file for more examples of a bit more complex queries.
 
-## Migrating from `sql_metadata` 1.x
-
-`sql_metadata.compat` module has been implemented to make the introduction of sql-metadata v2.0 smoother.
+## Migrating from `sql_metadata` 1.x / 2.x
 
-You can use it by simply changing the imports in your code from:
+The `sql_metadata.compat` module (previously provided for v1 → v2 migration) has been **removed in v3**.  Port your code to the class-based `Parser` API shown in the examples above:
 
-```python
-from sql_metadata import get_query_columns, get_query_tables
-```
-
-into:
-
-```python
-from sql_metadata.compat import get_query_columns, get_query_tables
-```
+| Old v1 helper                 | v3 replacement                            |
+|-------------------------------|-------------------------------------------|
+| `generalize_sql(sql)`         | `Parser(sql).generalize`                  |
+| `get_query_columns(sql)`      | `Parser(sql).columns`                     |
+| `get_query_tables(sql)`       | `Parser(sql).tables`                      |
+| `get_query_limit_and_offset(sql)` | `Parser(sql).limit_and_offset`        |
+| `get_query_tokens(sql)`       | `Parser(sql).tokens`                      |
+| `preprocess_query(sql)`       | `Parser(sql).query`                       |
 
-The following functions from the old API are available in the `sql_metadata.compat` module:
+For v2 → v3 users, the public `Parser` API is unchanged except:
 
-* `generalize_sql`
-* `get_query_columns` (since #131 columns aliases ARE NOT returned by this function)
-* `get_query_limit_and_offset`
-* `get_query_tables`
-* `get_query_tokens`
-* `preprocess_query`
+* The parsing engine is now [sqlglot](https://github.com/tobymao/sqlglot), which may normalise the *casing* and *spacing* of rendered CTE/subquery bodies (see the `with_queries` / `subqueries` examples above).
+* Malformed SQL now raises `InvalidQueryDefinition` (a `ValueError` subclass) instead of a plain `ValueError` — existing `except ValueError:` handlers continue to work.
 
 ## Authors and contributors
 

sql_metadata-3.0.0/pyproject.toml

@@ -0,0 +1,54 @@
+[tool.poetry]
+name = "sql_metadata"
+version = "3.0.0"
+license="MIT"
+description = "Uses sqlglot to parse SQL queries and extract metadata"
+authors = ["Maciej Brencz <maciej.brencz@gmail.com>", "Radosław Drążkiewicz <collerek@gmail.com>"]
+readme = "README.md"
+homepage = "https://github.com/macbre/sql-metadata"
+repository = "https://github.com/macbre/sql-metadata"
+
+packages = [
+    { include="sql_metadata" }
+]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+sqlglot = "^30.0.3"
+
+[tool.poetry.group.dev.dependencies]
+coverage = {extras = ["toml"], version = "^7.13"}
+pytest = "^9.0.3"
+pytest-cov = "^7.1.0"
+ruff = "^0.11"
+mypy = "^1.19"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.ruff]
+line-length = 88
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "C90", "I"]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 8
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_configs = true
+check_untyped_defs = true
+disallow_untyped_defs = true
+disallow_any_generics = true
+ignore_missing_imports = true
+
+[tool.coverage.run]
+relative_files = true
+
+[tool.coverage.report]
+show_missing = true
+fail_under = 100

sql_metadata-3.0.0/sql_metadata/__init__.py

@@ -0,0 +1,22 @@
+"""Parse SQL queries and extract structural metadata.
+
+The ``sql-metadata`` package analyses SQL statements and returns the
+tables, columns, aliases, CTE definitions, subqueries, values, comments,
+and query type they contain.  The primary entry point is :class:`Parser`::
+
+    from sql_metadata import Parser
+
+    parser = Parser("SELECT id, name FROM users WHERE active = 1")
+    print(parser.tables)   # ['users']
+    print(parser.columns)  # ['id', 'name', 'active']
+
+Under the hood the library delegates to `sqlglot <https://github.com/tobymao/sqlglot>`_
+for AST construction and tokenization, with custom dialect handling for
+MSSQL, MySQL, Hive/Spark, and TSQL bracket notation.
+"""
+
+from sql_metadata.exceptions import InvalidQueryDefinition
+from sql_metadata.keywords_lists import QueryType
+from sql_metadata.parser import Parser
+
+__all__ = ["InvalidQueryDefinition", "Parser", "QueryType"]

sql_metadata-3.0.0/sql_metadata/ast_parser.py

@@ -0,0 +1,117 @@
+"""Wrap ``sqlglot.parse()`` to produce an AST from raw SQL strings.
+
+Thin orchestrator that composes :class:`~sql_cleaner.SqlCleaner` (raw SQL
+preprocessing) and :class:`~dialect_parser.DialectParser` (dialect
+detection, parsing, quality validation) so that downstream extractors
+always receive a clean ``sqlglot.exp.Expression`` tree (or ``None`` /
+``ValueError``).
+"""
+
+from sqlglot import exp
+from sqlglot.dialects.dialect import DialectType
+
+from sql_metadata.dialect_parser import DialectParser
+from sql_metadata.sql_cleaner import SqlCleaner
+
+
+class ASTParser:
+    """Lazy wrapper around SQL parsing with dialect auto-detection.
+
+    Instantiated once per :class:`Parser` with the raw SQL string.  The
+    actual parsing is deferred until :attr:`ast` is first accessed, at
+    which point the SQL is cleaned and parsed through one or more sqlglot
+    dialects until a satisfactory AST is obtained.
+
+    :param sql: Raw SQL query string.
+    :type sql: str
+    """
+
+    def __init__(self, sql: str) -> None:
+        self._raw_sql = sql
+        self._ast: exp.Expression | None = None
+        self._dialect: DialectType = None
+        self._parsed = False
+        self._is_replace = False
+        self._cte_name_map: dict[str, str] = {}
+
+    @property
+    def ast(self) -> exp.Expression | None:
+        """The sqlglot AST for the query, lazily parsed on first access.
+
+        :returns: Root AST node, or ``None`` for empty/comment-only queries.
+        :rtype: exp.Expression
+        :raises ValueError: If the SQL is malformed and cannot be parsed.
+        """
+        if self._parsed:
+            return self._ast
+        self._parsed = True
+        self._ast = self._parse(self._raw_sql)
+        return self._ast
+
+    def _ensure_parsed(self) -> None:
+        """Trigger lazy parsing so side-effect fields are populated."""
+        _ = self.ast
+
+    @property
+    def dialect(self) -> DialectType:
+        """The sqlglot dialect that produced the current AST.
+
+        Set as a side-effect of :attr:`ast` access.  May be ``None``
+        (default dialect), a string like ``"mysql"``, or a custom
+        :class:`Dialect` subclass such as :class:`HashVarDialect`.
+
+        :rtype: DialectType
+        """
+        self._ensure_parsed()
+        return self._dialect
+
+    @property
+    def is_replace(self) -> bool:
+        """Whether the original query was a ``REPLACE INTO`` statement.
+
+        ``REPLACE INTO`` is rewritten to ``INSERT INTO`` before parsing
+        (sqlglot otherwise produces an opaque ``Command`` node).  This
+        flag allows :attr:`Parser.query_type` to restore the correct
+        :class:`QueryType.REPLACE` value.
+
+        :rtype: bool
+        """
+        self._ensure_parsed()
+        return self._is_replace
+
+    @property
+    def cte_name_map(self) -> dict[str, str]:
+        """Map of placeholder CTE names back to their original qualified form.
+
+        Keys are underscore-separated placeholders (``db__DOT__name``),
+        values are the original dotted names (``db.name``).
+
+        :rtype: dict[str, str]
+        """
+        self._ensure_parsed()
+        return self._cte_name_map
+
+    def _parse(self, sql: str) -> exp.Expression | None:
+        """Parse *sql* into a sqlglot AST.
+
+        Delegates preprocessing to :class:`SqlCleaner` and dialect
+        detection / parsing to :class:`DialectParser`.
+
+        :param sql: Raw SQL string (may include comments).
+        :type sql: str
+        :returns: Root AST node, or ``None`` for empty input.
+        :rtype: exp.Expression | None
+        :raises ValueError: If the SQL is malformed.
+        """
+        if not sql or not sql.strip():
+            return None
+
+        result = SqlCleaner.clean(sql)
+        if result.sql is None:
+            return None
+
+        self._is_replace = result.is_replace
+        self._cte_name_map = result.cte_name_map
+
+        ast, self._dialect = DialectParser().parse(result.sql)
+        return ast