sql-metadata 2.8.0__tar.gz → 2.18.0__tar.gz

{sql_metadata-2.8.0 → sql_metadata-2.18.0}/PKG-INFO

@@ -1,19 +1,22 @@
-Metadata-Version: 2.1
-Name: sql-metadata
-Version: 2.8.0
+Metadata-Version: 2.4
+Name: sql_metadata
+Version: 2.18.0
 Summary: Uses tokenized query returned by python-sqlparse and generates query metadata
-Home-page: https://github.com/macbre/sql-metadata
 License: MIT
+License-File: LICENSE
 Author: Maciej Brencz
 Author-email: maciej.brencz@gmail.com
-Requires-Python: >=3.7.2,<4.0.0
+Requires-Python: >=3.9,<4.0
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: sqlparse (>=0.4.1,<0.5.0)
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: sqlparse (>=0.4.1,<0.6.0)
+Project-URL: Homepage, https://github.com/macbre/sql-metadata
 Project-URL: Repository, https://github.com/macbre/sql-metadata
 Description-Content-Type: text/markdown
 

{sql_metadata-2.8.0 → sql_metadata-2.18.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sql_metadata"
-version = "2.8.0"
+version = "2.18.0"
 license="MIT"
 description = "Uses tokenized query returned by python-sqlparse and generates query metadata"
 authors = ["Maciej Brencz <maciej.brencz@gmail.com>", "Radosław Drążkiewicz <collerek@gmail.com>"]
@@ -13,17 +13,16 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.7.2"
-sqlparse = "^0.4.1"
+python = "^3.9"
+sqlparse = ">=0.4.1,<0.6.0"
 
 [tool.poetry.dev-dependencies]
-black = "^23.3"
-coverage = {extras = ["toml"], version = "^6.5"}
-pylint = "^2.17.2"
-pytest = "^7.3.1"
-pytest-cov = "^4.0.0"
-coveralls = "^3.3.1"
-flake8 = "^5.0.4"
+black = "^25.9"
+coverage = {extras = ["toml"], version = "^7.10"}
+pylint = "^3.3.9"
+pytest = "^8.4.2"
+pytest-cov = "^7.0.0"
+flake8 = "^7.3.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

{sql_metadata-2.8.0 → sql_metadata-2.18.0}/sql_metadata/__init__.py

@@ -2,6 +2,7 @@
 Module for parsing sql queries and returning columns,
 tables, names of with statements etc.
 """
+
 # pylint:disable=unsubscriptable-object
 from sql_metadata.parser import Parser
 from sql_metadata.keywords_lists import QueryType

{sql_metadata-2.8.0 → sql_metadata-2.18.0}/sql_metadata/compat.py

@@ -11,6 +11,7 @@ into:
 from sql_metadata.compat import get_query_columns, get_query_tables
 
 """
+
 # pylint:disable=missing-function-docstring
 from typing import List, Optional, Tuple
 

{sql_metadata-2.8.0 → sql_metadata-2.18.0}/sql_metadata/generalizator.py

@@ -1,6 +1,7 @@
 """
 Module used to produce generalized sql out of given query
 """
+
 import re
 import sqlparse
 
@@ -47,7 +48,8 @@ class Generalizator:
         :rtype: str
         """
         sql = sqlparse.format(self._raw_query, strip_comments=True)
-        sql = re.sub(r"\s{2,}", " ", sql)
+        sql = sql.replace("\n", " ")
+        sql = re.sub(r"[ \t]+", " ", sql)
         return sql
 
     @property

{sql_metadata-2.8.0 → sql_metadata-2.18.0}/sql_metadata/keywords_lists.py

@@ -9,6 +9,7 @@ from enum import Enum
 KEYWORDS_BEFORE_COLUMNS = {
     "SELECT",
     "WHERE",
+    "HAVING",
     "ORDERBY",
     "GROUPBY",
     "ON",
@@ -28,6 +29,7 @@ TABLE_ADJUSTMENT_KEYWORDS = {
     "RIGHTJOIN",
     "LEFTOUTERJOIN",
     "RIGHTOUTERJOIN",
+    "NATURALJOIN",
     "INTO",
     "UPDATE",
     "TABLE",
@@ -48,6 +50,7 @@ SUBQUERY_PRECEDING_KEYWORDS = {
     "RIGHTJOIN",
     "LEFTOUTERJOIN",
     "RIGHTOUTERJOIN",
+    "NATURALJOIN",
 }
 
 # section of a query in which column can exists
@@ -55,12 +58,14 @@ SUBQUERY_PRECEDING_KEYWORDS = {
 COLUMNS_SECTIONS = {
     "SELECT": "select",
     "WHERE": "where",
+    "HAVING": "having",
     "ORDERBY": "order_by",
     "ON": "join",
     "USING": "join",
     "INTO": "insert",
     "SET": "update",
     "GROUPBY": "group_by",
+    "INNERJOIN": "inner_join",
 }
 
 
@@ -77,6 +82,7 @@ class QueryType(str, Enum):
     CREATE = "CREATE TABLE"
     ALTER = "ALTER TABLE"
     DROP = "DROP TABLE"
+    TRUNCATE = "TRUNCATE TABLE"
 
 
 class TokenType(str, Enum):
@@ -102,8 +108,11 @@ SUPPORTED_QUERY_TYPES = {
     "DELETE": QueryType.DELETE,
     "WITH": QueryType.SELECT,
     "CREATETABLE": QueryType.CREATE,
+    "CREATETEMPORARY": QueryType.CREATE,
     "ALTERTABLE": QueryType.ALTER,
     "DROPTABLE": QueryType.DROP,
+    "CREATEFUNCTION": QueryType.CREATE,
+    "TRUNCATETABLE": QueryType.TRUNCATE,
 }
 
 # all the keywords we care for - rest is ignored in assigning
@@ -118,6 +127,7 @@ RELEVANT_KEYWORDS = {
     "RETURNING",
     "VALUES",
     "INDEX",
+    "KEY",
     "WITH",
     "WINDOW",
 }

{sql_metadata-2.8.0 → sql_metadata-2.18.0}/sql_metadata/parser.py

@@ -67,6 +67,7 @@ class Parser:  # pylint: disable=R0902
         self._nested_level = 0
         self._parenthesis_level = 0
         self._open_parentheses: List[SQLToken] = []
+        self._preceded_keywords: List[SQLToken] = []
         self._aliases_to_check = None
         self._is_in_nested_function = False
         self._is_in_with_block = False
@@ -113,25 +114,32 @@ class Parser:  # pylint: disable=R0902
             )
             .position
         )
-        if tokens[index].normalized in ["CREATE", "ALTER", "DROP"]:
+        if tokens[index].normalized == "CREATE":
+            switch = self._get_switch_by_create_query(tokens, index)
+        elif tokens[index].normalized in ("ALTER", "DROP", "TRUNCATE"):
             switch = tokens[index].normalized + tokens[index + 1].normalized
         else:
             switch = tokens[index].normalized
         self._query_type = SUPPORTED_QUERY_TYPES.get(switch, "UNSUPPORTED")
         if self._query_type == "UNSUPPORTED":
-            self._logger.error("Not supported query type: %s", self._raw_query)
+            # do not log the full query
+            # https://github.com/macbre/sql-metadata/issues/543
+            shorten_query = " ".join(self._raw_query.split(" ")[:3])
+
+            self._logger.error("Not supported query type: %s", shorten_query)
             raise ValueError("Not supported query type!")
         return self._query_type
 
     @property
-    def tokens(self) -> List[SQLToken]:
+    def tokens(self) -> List[SQLToken]:  # noqa: C901
         """
         Tokenizes the query
         """
         if self._tokens is not None:
             return self._tokens
 
-        parsed = sqlparse.parse(self._query)
+        # allow parser to be overriden
+        parsed = self._parse(self._query)
         tokens = []
         # handle empty queries (#12)
         if not parsed:
@@ -164,6 +172,8 @@ class Parser:  # pylint: disable=R0902
             elif token.is_right_parenthesis:
                 token.token_type = TokenType.PARENTHESIS
                 self._determine_closing_parenthesis_type(token=token)
+                if token.is_subquery_end:
+                    last_keyword = self._preceded_keywords.pop()
 
             last_keyword = self._determine_last_relevant_keyword(
                 token=token, last_keyword=last_keyword
@@ -214,7 +224,7 @@ class Parser:  # pylint: disable=R0902
                     self._handle_column_save(token=token, columns=columns)
 
                 elif token.is_column_name_inside_insert_clause:
-                    column = str(token.value).strip("`")
+                    column = str(token.value)
                     self._add_to_columns_subsection(
                         keyword=token.last_keyword_normalized, column=column
                     )
@@ -364,10 +374,8 @@ class Parser:  # pylint: disable=R0902
                     and self.query_type == "INSERT"
                 ):
                     continue
-
-                table_name = str(token.value.strip("`"))
                 token.token_type = TokenType.TABLE
-                tables.append(table_name)
+                tables.append(str(token.value))
 
         self._tables = tables - with_names
         return self._tables
@@ -460,7 +468,8 @@ class Parser:  # pylint: disable=R0902
                         while token.next_token and not token.is_with_query_end:
                             token = token.next_token
                         is_end_of_with_block = (
-                            token.next_token_not_comment.normalized
+                            token.next_token_not_comment is None
+                            or token.next_token_not_comment.normalized
                             in WITH_ENDING_KEYWORDS
                         )
                         if is_end_of_with_block:
@@ -501,7 +510,7 @@ class Parser:  # pylint: disable=R0902
                 True, value_attribute="is_with_query_end", direction="right"
             )
             query_token = with_start.next_token
-            while query_token != with_end:
+            while query_token is not None and query_token != with_end:
                 current_with_query.append(query_token)
                 query_token = query_token.next_token
             with_query_text = "".join([x.stringified_token for x in current_with_query])
@@ -530,12 +539,16 @@ class Parser:  # pylint: disable=R0902
                 ):
                     current_subquery.append(inner_token)
                     inner_token = inner_token.next_token
+
+                query_name = None
                 if inner_token.next_token.value in self.subqueries_names:
                     query_name = inner_token.next_token.value
-                else:
+                elif inner_token.next_token.is_as_keyword:
                     query_name = inner_token.next_token.next_token.value
+
                 subquery_text = "".join([x.stringified_token for x in current_subquery])
-                subqueries[query_name] = subquery_text
+                if query_name is not None:
+                    subqueries[query_name] = subquery_text
 
             token = token.next_token
 
@@ -619,7 +632,7 @@ class Parser:  # pylint: disable=R0902
         """
         Removes comments from SQL query
         """
-        return Generalizator(self.query).without_comments
+        return Generalizator(self._raw_query).without_comments
 
     @property
     def generalize(self) -> str:
@@ -660,6 +673,10 @@ class Parser:  # pylint: disable=R0902
             token.is_with_columns_end = True
             token.is_nested_function_end = False
             start_token = token.find_nearest_token("(")
+            # like: with (col1, col2) as (subquery) as ..., it enters an infinite loop.
+            # return exception
+            if start_token.is_with_query_start:
+                raise ValueError("This query is wrong")
             start_token.is_with_columns_start = True
             start_token.is_nested_function_start = False
             prev_token = start_token.previous_token
@@ -795,7 +812,8 @@ class Parser:  # pylint: disable=R0902
         return column if isinstance(column, list) else [column]
 
     @staticmethod
-    def _resolve_nested_query(
+    # pylint:disable=too-many-return-statements
+    def _resolve_nested_query(  # noqa: C901
         subquery_alias: str,
         nested_queries_names: List[str],
         nested_queries: Dict,
@@ -831,6 +849,9 @@ class Parser:  # pylint: disable=R0902
             # handle case when column name is used but subquery select all by wildcard
             if "*" in subparser.columns:
                 return column_name
+            for table in subparser.tables:
+                if f"{table}.*" in subparser.columns:
+                    return column_name
             raise exc  # pragma: no cover
         resolved_column = subparser.columns[column_index]
         return [resolved_column]
@@ -856,12 +877,13 @@ class Parser:  # pylint: disable=R0902
             # inside subquery / derived table
             token.is_subquery_start = True
             self._subquery_level += 1
+            self._preceded_keywords.append(token.last_keyword_normalized)
             token.subquery_level = self._subquery_level
         elif token.previous_token.normalized in KEYWORDS_BEFORE_COLUMNS.union({","}):
             # we are in columns and in a column subquery definition
             token.is_column_definition_start = True
         elif (
-            token.previous_token.is_as_keyword
+            token.previous_token_not_comment.is_as_keyword
             and token.last_keyword_normalized != "WINDOW"
         ):
             # window clause also contains AS keyword, but it is not a query
@@ -964,12 +986,13 @@ class Parser:  # pylint: disable=R0902
         # as double quotes are not properly handled in sqlparse
         query = re.sub(r"'.*?'", replace_quotes_in_string, self._raw_query)
         query = re.sub(r'"([^`]+?)"', r"`\1`", query)
-        query = re.sub(r'"([^`]+?)"\."([^`]+?)"', r"`\1`.`\2`", query)
         query = re.sub(r"'.*?'", replace_back_quotes_in_string, query)
 
         return query
 
     def _determine_last_relevant_keyword(self, token: SQLToken, last_keyword: str):
+        if token.value == "," and token.last_keyword_normalized == "ON":
+            return "FROM"
         if token.is_keyword and "".join(token.normalized.split()) in RELEVANT_KEYWORDS:
             if (
                 not (
@@ -993,6 +1016,8 @@ class Parser:  # pylint: disable=R0902
         Checks if token is a part of complex identifier like
         <schema>.<table>.<column> or <table/sub_query>.<column>
         """
+        if token.is_keyword:
+            return False
         return str(token) == "." or (
             index + 1 < self.tokens_length
             and str(self.non_empty_tokens[index + 1]) == "."
@@ -1006,16 +1031,19 @@ class Parser:  # pylint: disable=R0902
         is_complex = True
         while is_complex:
             value, is_complex = self._combine_tokens(index=index, value=value)
-            index = index - 2
+            index = index - 1
         token.value = value
 
     def _combine_tokens(self, index: int, value: str) -> Tuple[str, bool]:
         """
         Checks if complex identifier is longer and follows back until it's finished
         """
-        if index > 1 and str(self.non_empty_tokens[index - 1]) == ".":
-            prev_value = self.non_empty_tokens[index - 2].value.strip("`").strip('"')
-            value = f"{prev_value}.{value}"
+        if index > 1:
+            prev_value = self.non_empty_tokens[index - 1]
+            if not self._is_token_part_of_complex_identifier(prev_value, index - 1):
+                return value, False
+            prev_value = str(prev_value).strip("`")
+            value = f"{prev_value}{value}"
             return value, True
         return value, False
 
@@ -1065,3 +1093,26 @@ class Parser:  # pylint: disable=R0902
                             yield tok
             else:
                 yield token
+
+    @staticmethod
+    def _get_switch_by_create_query(tokens: List[SQLToken], index: int) -> str:
+        """
+        Return the switch that creates query type.
+        """
+        switch = tokens[index].normalized + tokens[index + 1].normalized
+
+        # Hive CREATE FUNCTION
+        if any(
+            index + i < len(tokens) and tokens[index + i].normalized == "FUNCTION"
+            for i in (1, 2)
+        ):
+            switch = "CREATEFUNCTION"
+
+        return switch
+
+    @staticmethod
+    def _parse(sql: str) -> Tuple[sqlparse.sql.Statement]:
+        """
+        Parse the SQL query using sqlparse library
+        """
+        return sqlparse.parse(sql)

{sql_metadata-2.8.0 → sql_metadata-2.18.0}/sql_metadata/token.py

@@ -1,6 +1,7 @@
 """
 Module contains internal SQLToken that creates linked list
 """
+
 from typing import Dict, List, Union
 
 import sqlparse.sql
@@ -186,7 +187,7 @@ class SQLToken:  # pylint: disable=R0902, R0904
         """
         return (
             self.next_token.normalized in [",", "FROM"]
-            and self.previous_token.normalized not in [",", ".", "(", "SELECT"]
+            and self.previous_token.normalized not in ["*", ",", ".", "(", "SELECT"]
             and not self.previous_token.is_keyword
             and (
                 self.last_keyword_normalized == "SELECT"
@@ -369,6 +370,15 @@ class SQLToken:  # pylint: disable=R0902, R0904
             return self.next_token.next_token_not_comment
         return self.next_token
 
+    @property
+    def previous_token_not_comment(self):
+        """
+        Property returning previous non-comment token
+        """
+        if self.previous_token and self.previous_token.is_comment:
+            return self.previous_token.previous_token_not_comment
+        return self.previous_token
+
     def is_constraint_definition_inside_create_table_clause(
         self, query_type: str
     ) -> bool:

{sql_metadata-2.8.0 → sql_metadata-2.18.0}/sql_metadata/utils.py

@@ -1,6 +1,7 @@
 """
 Module with various utils
 """
+
 from typing import Any, List, Sequence