sql-error-categorizer 0.2.4__tar.gz → 0.3.1__tar.gz

{sql_error_categorizer-0.2.4 → sql_error_categorizer-0.3.1}/.gitignore

@@ -1,7 +1,7 @@
 # CUSTOM
 test_q.sql
 test_s?.sql
-
+.codex
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

{sql_error_categorizer-0.2.4 → sql_error_categorizer-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql_error_categorizer
-Version: 0.2.4
+Version: 0.3.1
 Summary: This project analyses SQL statements and labels possible errors or complications.
 Project-URL: Repository, https://github.com/DavidePonzini/sql_error_categorizer
 Project-URL: Documentation, https://sql-error-categorizer.readthedocs.io/en/latest/index.html
@@ -14,10 +14,10 @@ Requires-Python: >=3.11
 Requires-Dist: psycopg2
 Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
-Requires-Dist: sql-error-taxonomy
+Requires-Dist: sql-error-taxonomy>=2.0.0
 Requires-Dist: sqlglot
 Requires-Dist: sqlparse
-Requires-Dist: sqlscope>=1.0.15
+Requires-Dist: sqlscope>=1.0.16
 Requires-Dist: z3-solver
 Description-Content-Type: text/markdown
 

{sql_error_categorizer-0.2.4 → sql_error_categorizer-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sql_error_categorizer"
-version = "0.2.4"
+version = "0.3.1"
 authors = [
   { name="Davide Ponzini", email="davide.ponzini95@gmail.com" },
 ]
@@ -21,8 +21,8 @@ dependencies = [
     "pyyaml",
     "sqlparse",
     "sqlglot",
-    "sqlscope>=1.0.15",
-    "sql_error_taxonomy",
+    "sqlscope>=1.0.16",
+    "sql_error_taxonomy>=2.0.0",
     "z3-solver",
     "python-dateutil",
 ]

{sql_error_categorizer-0.2.4 → sql_error_categorizer-0.3.1}/requirements.txt

@@ -4,8 +4,8 @@ sqlparse
 sqlglot==28.0.0
 z3-solver
 python-dateutil
-sql_error_taxonomy>=1.0.1
-sqlscope>=1.0.15
+sql_error_taxonomy>=2.0.0
+sqlscope>=1.0.16
 
 # Dependencies for development
 ipython

{sql_error_categorizer-0.2.4 → sql_error_categorizer-0.3.1}/src/sql_error_categorizer/detectors/complications.py

@@ -1,14 +1,10 @@
 '''Detector for complications in SQL queries.'''
 
-import difflib
-import re
-import sqlparse
-import sqlparse.keywords
 from typing import Callable
 from sqlglot import exp
 from sql_error_taxonomy import SqlErrors
-from sqlscope.catalog import ConstraintType, ConstraintColumn, Constraint
-from sqlscope import Query, Catalog
+from sqlscope.catalog import ConstraintType, ConstraintColumn
+from sqlscope import Query
 from sqlscope import util
 
 from .base import BaseDetector, DetectedError
@@ -36,29 +32,31 @@ class ComplicationDetector(BaseDetector):
         results: list[DetectedError] = super().run()
 
         checks = [
-            self.com_83_unnecessary_distinct_in_select_clause,
-            self.com_84_unnecessary_join,
-            self.com_85_unused_correlation_name,
-            self.com_86_correlation_names_are_always_identical,
-            self.com_87_unnecessary_general_comparison_operator,
-            self.com_88_like_without_wildcards,
-            self.com_89_unnecessarily_complicated_select_in_exists_subquery,
-            self.com_90_in_exists_can_be_replaced_by_comparison,
-            self.com_91_unnecessary_aggregate_function,
-            self.com_92_unnecessary_distinct_in_aggregate_function,
-            self.com_93_unnecessary_argument_of_count,
-            self.com_94_unnecessary_group_by_in_exists_subquery,
-            self.com_95_group_by_with_singleton_groups,
-            self.com_96_group_by_with_only_a_single_group,
-            self.com_97_group_by_can_be_replaced_by_distinct,
-            self.com_98_union_can_be_replaced_by_or,
-            self.com_99_complication_unnecessary_column_in_order_by_clause,
-            self.com_100_order_by_in_subquery,
-            self.com_101_inefficient_having,
-            self.com_102_inefficient_union,
-            self.com_103_condition_in_the_subquery_can_be_moved_up,
-            self.com_104_condition_on_left_table_in_left_outer_join,
-            self.com_105_outer_join_can_be_replaced_by_inner_join,
+            self.detect_82_unnecessary_complication,                                # ok
+            self.detect_83_unnecessary_distinct_in_select_clause,                   # ok
+            self.detect_84_unnecessary_table_reference,                             # TODO: refactor/implement
+            self.detect_85_unused_correlation_name,                                 # TODO: implement
+            self.detect_86_tables_have_same_data,                                   # TODO: implement
+            self.detect_125_correlation_name_identical_to_table_name,               # TODO: implement
+            self.detect_87_unnecessary_general_comparison_operator,                 # TODO: implement
+            self.detect_88_like_without_wildcards,                                  # ok
+            self.detect_89_unnecessarily_complicated_select_in_exists_subquery,     # TODO: implement
+            self.detect_90_in_exists_can_be_replaced_by_comparison,                 # TODO: implement
+            self.detect_91_unnecessary_aggregate_function,                          # TODO: implement
+            self.detect_92_unnecessary_distinct_in_aggregate_function,              # ok
+            self.detect_93_unnecessary_argument_of_count,                           # ok
+            self.detect_94_unnecessary_group_by_in_exists_subquery,                 # TODO: implement
+            self.detect_95_group_by_with_singleton_groups,                          # ok
+            self.detect_96_group_by_with_only_a_single_group,                       # TODO: implement
+            self.detect_97_group_by_can_be_replaced_by_distinct,                    # ok
+            self.detect_98_union_can_be_replaced_by_or,                             # TODO: implement
+            self.detect_99_unnecessary_column_in_order_by_clause,                   # TODO: refactor/implement
+            self.detect_100_order_by_in_subquery,                                   # TODO: implement
+            self.detect_101_inefficient_having,                                     # TODO: implement
+            self.detect_102_inefficient_union,                                      # TODO: implement
+            self.detect_103_condition_in_the_subquery_can_be_moved_up,              # TODO: implement
+            self.detect_104_outer_join_can_be_replaced_by_inner_join,               # TODO: implement
+            self.detect_126_unused_cte,                                             #
         ]
         
         for chk in checks:
@@ -66,7 +64,11 @@ class ComplicationDetector(BaseDetector):
 
         return results
 
-    def com_83_unnecessary_distinct_in_select_clause(self) -> list[DetectedError]:
+    def detect_82_unnecessary_complication(self) -> list[DetectedError]:
+        '''NOTE: this is an umbrella term, so it can't be directly detected.'''
+        return []
+
+    def detect_83_unnecessary_distinct_in_select_clause(self) -> list[DetectedError]:
         '''
         Flags a SELECT DISTINCT clause that is unnecessary because the selected
         columns are already unique due to existing constraints.
@@ -81,12 +83,11 @@ class ComplicationDetector(BaseDetector):
             constraints = [c for c in select.output.unique_constraints if c.constraint_type != ConstraintType.DISTINCT]
 
             if len(constraints) > 0:
-                result.append(DetectedError(SqlErrors.COM_83_UNNECESSARY_DISTINCT_IN_SELECT_CLAUSE, (select.sql,)))
+                result.append(DetectedError(SqlErrors.UNNECESSARY_DISTINCT_IN_SELECT_CLAUSE, (select.sql,)))
 
         return result
 
-    # TODO: refactor
-    def com_84_unnecessary_join(self) -> list[DetectedError]:
+    def detect_84_unnecessary_table_reference(self) -> list[DetectedError]:
         '''
         Flags a query that joins to a table not present in the correct solution.
         '''
@@ -110,25 +111,25 @@ class ComplicationDetector(BaseDetector):
                 # Find the original table name (with alias if it was used) to report back
                 original_table_name = next((t for t in original_q_tables if t.lower().startswith(table_name_lower)), table_name_lower)
                 results.append((
-                    SqlErrors.COM_84_UNNECESSARY_JOIN,
+                    SqlErrors.UNNECESSARY_TABLE_REFERENCE,
                     f"Unnecessary JOIN: The table '{original_table_name}' is not needed to answer the query."
                 ))
             
         return results
     
-    # TODO: implement
-    def com_85_unused_correlation_name(self) -> list[DetectedError]:
+    def detect_85_unused_correlation_name(self) -> list[DetectedError]:
         return []
     
-    # TODO: implement
-    def com_86_correlation_names_are_always_identical(self) -> list[DetectedError]:
+    def detect_86_tables_have_same_data(self) -> list[DetectedError]:
+        return []
+
+    def detect_125_correlation_name_identical_to_table_name(self) -> list[DetectedError]:
         return []
     
-    # TODO: implement
-    def com_87_unnecessary_general_comparison_operator(self) -> list[DetectedError]:
+    def detect_87_unnecessary_general_comparison_operator(self) -> list[DetectedError]:
         return []
     
-    def com_88_like_without_wildcards(self) -> list[DetectedError]:
+    def detect_88_like_without_wildcards(self) -> list[DetectedError]:
         '''
         Flags queries where the LIKE operator is used without wildcards ('%' or '_').
         This indicates a potential misunderstanding, where the '=' operator should
@@ -157,23 +158,20 @@ class ComplicationDetector(BaseDetector):
                 if '%' not in pattern_value and '_' not in pattern_value:
                     full_expression = str(like)
 
-                    results.append(DetectedError(SqlErrors.COM_88_LIKE_WITHOUT_WILDCARDS, (full_expression,)))
+                    results.append(DetectedError(SqlErrors.LIKE_WITHOUT_WILDCARDS, (full_expression,)))
 
         return results
     
-    # TODO: implement
-    def com_89_unnecessarily_complicated_select_in_exists_subquery(self) -> list[DetectedError]:
+    def detect_89_unnecessarily_complicated_select_in_exists_subquery(self) -> list[DetectedError]:
         return []
     
-    # TODO: implement
-    def com_90_in_exists_can_be_replaced_by_comparison(self) -> list[DetectedError]:
+    def detect_90_in_exists_can_be_replaced_by_comparison(self) -> list[DetectedError]:
         return []
     
-    # TODO: implement
-    def com_91_unnecessary_aggregate_function(self) -> list[DetectedError]:
+    def detect_91_unnecessary_aggregate_function(self) -> list[DetectedError]:
         return []
     
-    def com_92_unnecessary_distinct_in_aggregate_function(self) -> list[DetectedError]:
+    def detect_92_unnecessary_distinct_in_aggregate_function(self) -> list[DetectedError]:
         '''MIN and MAX never require DISTINCT. For other aggregate functions, DISTINCT is unnecessary if the argument is unique.'''
 
         results: list[DetectedError] = []
@@ -189,7 +187,7 @@ class ComplicationDetector(BaseDetector):
                     continue
 
                 if isinstance(agg_func, (exp.Min, exp.Max)):
-                    results.append(DetectedError(SqlErrors.COM_92_UNNECESSARY_DISTINCT_IN_AGGREGATE_FUNCTION, (str(agg_func),)))
+                    results.append(DetectedError(SqlErrors.UNNECESSARY_DISTINCT_IN_AGGREGATE_FUNCTION, (str(agg_func),)))
                     continue
 
                 arg_expr = agg_func.this.expressions   # `.this` is the DISTINCT, `.expressions` are the arguments
@@ -199,7 +197,7 @@ class ComplicationDetector(BaseDetector):
                 for expr in arg_expr:
                     # Check if the argument is a constant literal
                     if isinstance(expr, exp.Literal):
-                        results.append(DetectedError(SqlErrors.COM_92_UNNECESSARY_DISTINCT_IN_AGGREGATE_FUNCTION, (str(agg_func),)))
+                        results.append(DetectedError(SqlErrors.UNNECESSARY_DISTINCT_IN_AGGREGATE_FUNCTION, (str(agg_func),)))
                         continue
 
                     # Check if the argument is a column
@@ -210,18 +208,17 @@ class ComplicationDetector(BaseDetector):
                         unique_constraints = [c for c in select.all_constraints if c.constraint_type == ConstraintType.UNIQUE]
                         for constraint in unique_constraints:
                             if { ConstraintColumn(column_name, table_idx=select._get_table_idx_for_column(expr)) } == constraint.columns:
-                                results.append(DetectedError(SqlErrors.COM_92_UNNECESSARY_DISTINCT_IN_AGGREGATE_FUNCTION, (str(agg_func),)))
+                                results.append(DetectedError(SqlErrors.UNNECESSARY_DISTINCT_IN_AGGREGATE_FUNCTION, (str(agg_func),)))
                                 break
         return results
     
-    def com_93_unnecessary_argument_of_count(self) -> list[DetectedError]:
+    def detect_93_unnecessary_argument_of_count(self) -> list[DetectedError]:
         return []
     
-    # TODO: implement
-    def com_94_unnecessary_group_by_in_exists_subquery(self) -> list[DetectedError]:
+    def detect_94_unnecessary_group_by_in_exists_subquery(self) -> list[DetectedError]:
         return []
     
-    def com_95_group_by_with_singleton_groups(self) -> list[DetectedError]:
+    def detect_95_group_by_with_singleton_groups(self) -> list[DetectedError]:
         '''
         Flags GROUP BY clauses on singleton groups due to the presence
         of UNIQUE constraints on the grouped columns.
@@ -241,16 +238,15 @@ class ComplicationDetector(BaseDetector):
 
             for constraint in constraints:
                 if constraint.columns.issubset(group_by_constraint.columns):
-                    results.append(DetectedError(SqlErrors.COM_95_GROUP_BY_WITH_SINGLETON_GROUPS, (group_by_constraint, constraint)))
+                    results.append(DetectedError(SqlErrors.GROUP_BY_WITH_SINGLETON_GROUPS, (group_by_constraint, constraint)))
                     break        
 
         return results
     
-    # TODO: implement
-    def com_96_group_by_with_only_a_single_group(self) -> list[DetectedError]:
+    def detect_96_group_by_with_only_a_single_group(self) -> list[DetectedError]:
         return []
     
-    def com_97_group_by_can_be_replaced_by_distinct(self) -> list[DetectedError]:
+    def detect_97_group_by_can_be_replaced_by_distinct(self) -> list[DetectedError]:
         '''
         Flags GROUP BY clauses that can be replaced by SELECT DISTINCT.
         This occurs when all selected columns are included in the GROUP BY clause
@@ -290,18 +286,14 @@ class ComplicationDetector(BaseDetector):
             group_by_col_names = {(util.ast.column.get_real_name(col), select._get_table_idx_for_column(col)) for col in group_by_columns}
 
             if select_col_names == group_by_col_names:
-                results.append(DetectedError(SqlErrors.COM_97_GROUP_BY_CAN_BE_REPLACED_WITH_DISTINCT, (select_col_names,)))
+                results.append(DetectedError(SqlErrors.GROUP_BY_CAN_BE_REPLACED_WITH_DISTINCT, (select_col_names,)))
 
         return results
                         
-
-    
-    # TODO: implement
-    def com_98_union_can_be_replaced_by_or(self) -> list[DetectedError]:
+    def detect_98_union_can_be_replaced_by_or(self) -> list[DetectedError]:
         return []
     
-    # TODO: refactor
-    def com_99_complication_unnecessary_column_in_order_by_clause(self) -> list[DetectedError]:
+    def detect_99_unnecessary_column_in_order_by_clause(self) -> list[DetectedError]:
         '''
         Flags when the ORDER BY clause contains unnecessary columns in addition
         to the required ones.
@@ -329,8 +321,7 @@ class ComplicationDetector(BaseDetector):
 
         return results
     
-    # TODO: implement
-    def com_100_order_by_in_subquery(self) -> list[DetectedError]:
+    def detect_100_order_by_in_subquery(self) -> list[DetectedError]:
         '''
         Flags when a subquery contains an ORDER BY clause.
         Subqueries both ORDER BY and LIMIT are considered valid.
@@ -348,157 +339,37 @@ class ComplicationDetector(BaseDetector):
 
                 checked_subqueries.add(subquery.sql)
                 if subquery.order_by and not subquery.limit:
-                    results.append(DetectedError(SqlErrors.COM_100_ORDER_BY_IN_SUBQUERY, (subquery.sql,)))
+                    results.append(DetectedError(SqlErrors.ORDER_BY_IN_SUBQUERY, (subquery.sql,)))
 
         return results
     
-    # TODO: implement
-    def com_101_inefficient_having(self) -> list[DetectedError]:
-        return []
-    
-    # TODO: implement
-    def com_102_inefficient_union(self) -> list[DetectedError]:
+    def detect_101_inefficient_having(self) -> list[DetectedError]:
         return []
     
-    # TODO: implement
-    def com_103_condition_in_the_subquery_can_be_moved_up(self) -> list[DetectedError]:
+    def detect_102_inefficient_union(self) -> list[DetectedError]:
         return []
     
-    # TODO: implement
-    def com_104_condition_on_left_table_in_left_outer_join(self) -> list[DetectedError]:
+    def detect_103_condition_in_the_subquery_can_be_moved_up(self) -> list[DetectedError]:
         return []
     
-    # TODO: implement
-    def com_105_outer_join_can_be_replaced_by_inner_join(self) -> list[DetectedError]:
+    def detect_104_outer_join_can_be_replaced_by_inner_join(self) -> list[DetectedError]:
         return []
 
+    def detect_126_unused_cte(self) -> list[DetectedError]:
+        results: list[DetectedError] = []
 
-    #region Utility methods
-    def _get_select_columns(self, ast: dict) -> list:
-        '''
-        Extracts a list of simple column names from a SELECT query's AST.
-        '''
-        columns = []
-        if not ast:
-            return columns
-
-        select_expressions = ast.get('args', {}).get('expressions', [])
-        
-        for expr_node in select_expressions:
-            col_name = self._find_underlying_column(expr_node)
-            if col_name:
-                columns.append(col_name)
-        
-        return columns
-    def _find_underlying_column(self, node: dict):
-        '''
-        Recursively traverses an expression node to find the underlying column identifier.
-        '''
-        if not isinstance(node, dict):
-            return None
-        
-        node_class = node.get('class')
-
-        if node_class == 'Paren':
-            return self._find_underlying_column(node.get('args', {}).get('this'))
-
-        if node_class == 'Column':
-            try:
-                return node['args']['expression']['args']['this']
-            except (KeyError, TypeError):
-                try:
-                    return node['args']['this']['args']['this']
-                except (KeyError, TypeError):
-                    return None
-
-        if node_class == 'Alias':
-            return self._find_underlying_column(node.get('args', {}).get('this'))
-    def _get_from_tables(self, ast: dict, with_alias=False) -> list:
-        '''
-        Extracts a list of all table names from the FROM and JOIN clauses of a query's AST.
-        '''
-        tables = []
-        if not ast:
-            return tables
-        
-        args = ast.get('args', {})
-
-        # 1. Process the main table from the 'from' clause
-        from_node = args.get('from')
-        if from_node:
-            # The actual table data is inside the 'this' argument of the 'From' node
-            main_table_node = from_node.get('args', {}).get('this')
-            if main_table_node:
-                self._collect_tables_recursive(main_table_node, tables, with_alias)
-
-        # 2. Process all tables from the 'joins' list
-        join_nodes = args.get('joins', [])
-        for join_node in join_nodes:
-            self._collect_tables_recursive(join_node, tables, with_alias)
-                
-        return list(set(tables))
-    def _collect_tables_recursive(self, node: dict, tables: list, with_alias=False):
-        '''
-        Recursively traverses a FROM clause node (including joins) to collect table names.
-        '''
-        if not isinstance(node, dict):
-            return
-
-        node_class = node.get('class')
-
-        # This part handles aliased tables (e.g., "customer c") and regular tables
-        if node_class == 'Alias':
-            underlying_node = node.get('args', {}).get('this')
-            # Recurse in case the alias is on a subquery or another join
-            self._collect_tables_recursive(underlying_node, tables, with_alias)
-
-        elif node_class == 'Table':
-            try:
-                # The AST nests identifiers, so we go deep to get the name
-                table_name = node['args']['this']['args']['this']
-                alias_node = node.get('args', {}).get('alias')
-                if with_alias and alias_node:
-                    alias_name = alias_node.get('args', {}).get('this', {}).get('args', {}).get('this')
-                    tables.append(f"{table_name} AS {alias_name}")
-                else:
-                    tables.append(table_name)
-            except (KeyError, TypeError):
-                pass
+        if not self.query.ctes:
+            return results
         
-        # This part handles Join nodes found in the 'joins' list
-        elif node_class == 'Join':
-            # The joined table is in the 'this' argument of the Join node
-            self._collect_tables_recursive(node.get('args', {}).get('this'), tables, with_alias)
-            # The other side of the join is already handled in the 'from' clause,
-            # but we check for 'expression' for other potential join structures.
-            if 'expression' in node.get('args', {}):
-                self._collect_tables_recursive(node.get('args', {}).get('expression'), tables, with_alias)
-    def _get_orderby_columns(self, ast: dict) -> list:
-        '''
-        Extracts a list of columns and their sort direction from an ORDER BY clause.
-        '''
-        orderby_terms = []
-        if not ast:
-            return orderby_terms
+        used_ctes: dict[int, bool] = {i: False for i in range(len(self.query.ctes))}
 
-        orderby_node = ast.get('args', {}).get('order')
-        if not orderby_node:
-            return orderby_terms
+        for select in self.query.selects:
+            for table in select.referenced_tables:
+                if table.cte_idx is not None:
+                    used_ctes[table.cte_idx] = True
 
-        try:
-            for term_node in orderby_node['args']['expressions']:
-                if term_node.get('class') != 'Ordered':
-                    continue
-                
-                column_node = term_node.get('args', {}).get('this')
-                
-                col_name = self._find_underlying_column(column_node)
-                
-                if col_name:
-                    direction = term_node.get('args', {}).get('direction', 'ASC').upper()
-                    orderby_terms.append((col_name, direction))
-        except (KeyError, AttributeError):
-            return []
-            
-        return orderby_terms
-    #endregion Utility methods
+        for cte_idx, used in used_ctes.items():
+            if not used:
+                results.append(DetectedError(SqlErrors.UNUSED_CTE, (self.query.ctes[cte_idx].sql,)))
+
+        return results