sql-error-categorizer 0.1.0__py3-none-any.whl

sql_error_categorizer/__init__.py

@@ -0,0 +1,56 @@
+# Hidden, internal use only
+from .detectors import BaseDetector as _BaseDetector, Detector as _Detector
+
+# Public API
+from .sql_errors import SqlErrors
+from .catalog import Catalog, build_catalog, load_json as load_catalog
+from .detectors import SyntaxErrorDetector, SemanticErrorDetector, LogicalErrorDetector, ComplicationDetector, DetectedError
+
+def get_errors(query_str: str,
+               solutions: list[str] = [],
+               catalog: Catalog = Catalog(),
+               search_path: str = 'public',
+               solution_search_path: str = 'public',
+               detectors: list[type[_BaseDetector]] = [
+                   SyntaxErrorDetector,
+                   SemanticErrorDetector,
+                   LogicalErrorDetector,
+                   ComplicationDetector
+                ],
+               debug: bool = False) -> list[DetectedError]:
+    '''Detect SQL errors in the given query string.'''
+    det = _Detector(query_str,
+                    solutions=solutions,
+                    catalog=catalog,
+                    search_path=search_path,
+                    solution_search_path=solution_search_path,
+                    debug=debug)
+
+    for detector in detectors:
+        det.add_detector(detector)
+
+    return det.run()
+
+def get_error_types(query_str: str,
+                    solutions: list[str] = [],
+                    catalog: Catalog = Catalog(),
+                    search_path: str = 'public',
+                    solution_search_path: str = 'public',
+                    detectors: list[type[_BaseDetector]] = [
+                        SyntaxErrorDetector,
+                        SemanticErrorDetector,
+                        LogicalErrorDetector,
+                        ComplicationDetector
+                    ],
+                    debug: bool = False) -> set[SqlErrors]:
+    '''Detect SQL error types in the given query string.'''
+
+    detected_errors = get_errors(query_str,
+                                 solutions=solutions,
+                                 catalog=catalog,
+                                 search_path=search_path,
+                                 solution_search_path=solution_search_path,
+                                 detectors=detectors,
+                                 debug=debug)
+    
+    return {error.error for error in detected_errors}

sql_error_categorizer/catalog/__init__.py

@@ -0,0 +1,73 @@
+'''Builds a catalog of existing schemas/tables/columns by executing the provided SQL string in a temporary PostgreSQL database.'''
+
+from .catalog import Catalog, Schema, Table, Column, UniqueConstraintType
+import psycopg2
+import time
+from . import queries
+
+def build_catalog(sql_string: str, *, hostname: str, port: int, user: str, password: str, use_temp_schema: bool = True) -> Catalog:
+    '''Builds a catalog by executing the provided SQL string in a temporary PostgreSQL database.'''
+    result = Catalog()
+
+    if sql_string.strip() == '':
+        return result
+
+    conn = psycopg2.connect(host=hostname, port=port, user=user, password=password)
+    cur = conn.cursor()
+    
+    # Use a temporary schema with a fixed name
+    if use_temp_schema:
+        schema_name = f'sql_error_categorizer_{time.time_ns()}'
+        cur.execute(f'CREATE schema {schema_name};')
+        cur.execute(f'SET search_path TO {schema_name};')
+    else:
+        schema_name = '%'   # TODO: it's a bit hackish, find a more stable solution
+    
+    # Create the tables
+    cur.execute(sql_string)
+
+    from dav_tools import messages
+    messages.info(queries.COLUMNS(schema_name))
+
+    # Fetch the catalog information
+    cur.execute(queries.COLUMNS(schema_name))
+    columns_info = cur.fetchall()
+
+    messages.debug(f'Fetched {len(columns_info)} columns from the database.')
+
+    for column in columns_info:
+        messages.debug(f'Processing column: {column}')
+        schema_name, table_name, column_name, column_type, numeric_precision, numeric_scale, is_nullable, fk_schema, fk_table, fk_column = column
+
+        result.add_column(schema_name, table_name, column_name,
+                          column_type, numeric_precision, numeric_scale,
+                          is_nullable,
+                          fk_schema, fk_table, fk_column)
+
+    # Fetch unique constraints (including primary keys)
+    cur.execute(queries.UNIQUE_COLUMNS(schema_name))
+    unique_constraints_info = cur.fetchall()
+    for constraint in unique_constraints_info:
+        schema_name, table_name, constraint_type, columns = constraint
+        columns = set(columns.strip('{}').split(','))  # Postgres returns {col1,col2,...}
+
+        if constraint_type == 'PRIMARY KEY':
+            constraint_type = UniqueConstraintType.PRIMARY_KEY
+        elif constraint_type == 'UNIQUE':
+            constraint_type = UniqueConstraintType.UNIQUE
+        else:
+            raise ValueError(f'Unknown constraint type: {constraint_type}')
+
+        result[schema_name][table_name].add_unique_constraint(columns, constraint_type)
+
+    # Clean up
+    if use_temp_schema:
+        cur.execute(f'DROP schema {schema_name} CASCADE;')
+    conn.rollback()     # no need to save anything
+
+    return result
+
+def load_json(path: str) -> Catalog:
+    '''Loads a catalog from a JSON file.'''
+    return Catalog.load_json(path)
+

sql_error_categorizer/catalog/catalog.py

@@ -0,0 +1,328 @@
+from dataclasses import dataclass, field
+import json
+from typing import Self
+from enum import Enum
+from copy import deepcopy
+
+# region UniqueConstraint
+class UniqueConstraintType(Enum):
+    PRIMARY_KEY = 'PRIMARY KEY'
+    UNIQUE = 'UNIQUE'
+
+class UniqueConstraint:
+    def __init__(self, columns: set[str], constraint_type: UniqueConstraintType) -> None:
+        self.columns = columns
+        self.constraint_type = constraint_type
+
+    def __repr__(self, level: int = 0) -> str:
+        indent = '  ' * level
+        return f'{indent}UniqueConstraint({self.constraint_type.value}: {self.columns})'
+    
+    def to_dict(self) -> dict:
+        return {
+            'columns': list(self.columns),  # JSON-friendly (list)
+            'constraint_type': self.constraint_type.value,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> 'UniqueConstraint':
+        return cls(columns=set(c.lower() for c in data['columns']),
+                   constraint_type=UniqueConstraintType(data['constraint_type']))
+# endregion
+
+# region Column
+@dataclass
+class Column:
+    name: str
+    column_type: str = 'UNKNOWN'
+    numeric_precision: int | None = None
+    numeric_scale: int | None = None
+    is_nullable: bool = True
+    is_constant: bool = False
+    fk_schema: str | None = None
+    fk_table: str | None = None
+    fk_column: str | None = None
+
+    @property
+    def is_fk(self) -> bool:
+        return all([self.fk_schema, self.fk_table, self.fk_column])
+
+    def __repr__(self, level: int = 0) -> str:
+        indent = '  ' * level
+        return f'{indent}Column(name=\'{self.name}\', type=\'{self.column_type}\', is_fk={self.is_fk}, is_nullable={self.is_nullable}, is_constant={self.is_constant})'
+
+    def to_dict(self) -> dict:
+        return {
+            'name': self.name,
+            'column_type': self.column_type,
+            'numeric_precision': self.numeric_precision,
+            'numeric_scale': self.numeric_scale,
+            'is_nullable': self.is_nullable,
+            'fk_schema': self.fk_schema,
+            'fk_table': self.fk_table,
+            'fk_column': self.fk_column,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> 'Column':
+        return cls(
+            name=data['name'],
+            column_type=data['column_type'],
+            numeric_precision=data.get('numeric_precision'),
+            numeric_scale=data.get('numeric_scale'),
+            is_nullable=data.get('is_nullable', True),
+            fk_schema=(data.get('fk_schema') or None),
+            fk_table=(data.get('fk_table') or None),
+            fk_column=(data.get('fk_column') or None),
+        )
+# endregion
+
+# region Table
+@dataclass
+class Table:
+    '''A database table, with columns and unique constraints. Supports multiple columns with the same name (e.g. from joins).'''
+    name: str
+    unique_constraints: list[UniqueConstraint] = field(default_factory=list)
+    columns: list[Column] = field(default_factory=list)
+
+    def add_unique_constraint(self, columns: set[str], constraint_type: UniqueConstraintType) -> None:
+        self.unique_constraints.append(UniqueConstraint(columns, constraint_type))
+
+    def add_column(self,
+                   name: str,
+                   column_type: str,
+                   numeric_precision: int | None = None,
+                   numeric_scale: int | None = None,
+                   is_nullable: bool = True,
+                   is_constant: bool = False,
+                   fk_schema: str | None = None,
+                   fk_table: str | None = None,
+                   fk_column: str | None = None) -> Column:
+        column = Column(name=name,
+                        column_type=column_type,
+                        numeric_precision=numeric_precision,
+                        numeric_scale=numeric_scale,
+                        is_nullable=is_nullable,
+                        is_constant=is_constant,
+                        fk_schema=fk_schema,
+                        fk_table=fk_table,
+                        fk_column=fk_column)
+        self.columns.append(column)
+        return column
+    
+    def has_column(self, column_name: str) -> bool:
+        '''Checks if a column exists in the table.'''
+        return any(col.name == column_name for col in self.columns)
+
+    def __getitem__(self, column_name: str) -> Column:
+        '''Gets a column from the table, creating it if it does not exist.'''
+        for col in self.columns:
+            if col.name == column_name:
+                return col
+
+        new_col = Column(name=column_name)
+        self.columns.append(new_col)
+        return new_col
+
+    def __repr__(self, level: int = 0) -> str:
+        indent = '  ' * level
+
+        columns = '\n'.join([col.__repr__(level + 1) for col in self.columns])
+        if len(self.unique_constraints) < 2:
+            unique_constraints_str = ', '.join([uc.__repr__(0) for uc in self.unique_constraints])
+        else:
+            unique_constraints_str = '\n' + '\n'.join([uc.__repr__(level + 1) for uc in self.unique_constraints]) + '\n' + indent
+        
+        if len(self.columns) > 0:
+            columns = '\n' + columns + '\n' + indent
+        
+        return f'{indent}Table(name=\'{self.name}\', columns=[{columns}], unique_constraints=[{unique_constraints_str}])'
+
+    def to_dict(self) -> dict:
+        return {
+            'name': self.name,
+            'unique_constraints': [uc.to_dict() for uc in self.unique_constraints],
+            'columns': [col.to_dict() for col in self.columns],
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> 'Table':
+        table = cls(name=data['name'])
+        # Unique constraints first (so Column.is_pk works immediately on repr, etc.)
+        for uc_data in data.get('unique_constraints', []):
+            uc = UniqueConstraint.from_dict(uc_data)
+            table.unique_constraints.append(uc)
+        # Columns
+        for col_data in (data.get('columns') or []):
+            col = Column.from_dict(col_data)
+            # Keep internal store normalized to lowercase
+            table.columns.append(col)
+        return table
+# endregion
+
+# region Schema
+@dataclass
+class Schema:
+    name: str
+    _tables: dict[str, Table] = field(default_factory=dict)
+    functions: set[str] = field(default_factory=set)
+
+    def __getitem__(self, table_name: str) -> Table:
+        '''Gets a table from the schema, creating it if it does not exist.'''
+        if table_name not in self._tables:
+            self._tables[table_name] = Table(table_name)
+        return self._tables[table_name]
+
+    def __setitem__(self, table_name: str, table: Table) -> None:
+        '''Sets a table in the schema, replacing any existing table with the same name.'''
+        self._tables[table_name] = table
+    
+    def has_table(self, table_name: str) -> bool:
+        '''Checks if a table exists in the schema.'''
+        return table_name in self._tables
+    
+    def has_column(self, table_name: str, column_name: str) -> bool:
+        '''Checks if a column exists in the schema.'''
+        if not self.has_table(table_name):
+            return False
+        return self.__getitem__(table_name).has_column(column_name)
+
+    @property
+    def table_names(self) -> set[str]:
+        '''Returns all table names in the schema.'''
+        return set(self._tables.keys())
+
+    def __repr__(self, level: int = 0) -> str:
+        indent = '  ' * level
+        tables = '\n'.join([table.__repr__(level + 1) for table in self._tables.values()])
+        return f'{indent}Schema(name=\'{self.name}\', tables=[\n{tables}\n{indent}])'
+
+    def to_dict(self) -> dict:
+        return {
+            'name': self.name,
+            'tables': {name: tbl.to_dict() for name, tbl in self._tables.items()},
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> 'Schema':
+        schema = cls(name=data['name'])
+        for _, tbl_data in (data.get('tables') or {}).items():
+            tbl = Table.from_dict(tbl_data)
+            schema._tables[tbl.name] = tbl
+        return schema
+# endregion
+
+# region Catalog
+@dataclass
+class Catalog:
+    _schemas: dict[str, Schema] = field(default_factory=dict)
+
+    def __getitem__(self, schema_name: str) -> Schema:
+        '''Gets a schema from the catalog, creating it if it does not exist.'''
+
+        if schema_name not in self._schemas:
+            self._schemas[schema_name] = Schema(schema_name)
+        return self._schemas[schema_name]
+    
+    def __setitem__(self, schema_name: str, schema: Schema) -> Schema:
+        '''Sets a schema in the catalog, replacing any existing schema with the same name.'''
+        
+        self._schemas[schema_name] = schema
+        return schema
+    
+    def has_schema(self, schema_name: str) -> bool:
+        '''Checks if a schema exists in the catalog.'''
+        
+        return schema_name in self._schemas
+    
+    def copy_table(self, schema_name: str, table_name: str, table: Table) -> Table:
+        '''Copies a table into the catalog, creating the schema if it does not exist.'''
+        
+        new_table = deepcopy(table)
+        self[schema_name][table_name] = new_table
+        
+        return new_table
+
+    def has_table(self, schema_name: str, table_name: str) -> bool:
+        '''
+            Checks if a table exists in the specified schema in the catalog.
+
+            Returns False if the schema or table do not exist.
+        '''
+
+        if not self.has_schema(schema_name):
+            return False
+        return self.__getitem__(schema_name).has_table(table_name)
+
+    def add_column(self, schema_name: str, table_name: str, column_name: str,
+                   column_type: str, numeric_precision: int | None = None, numeric_scale: int | None = None,
+                   is_nullable: bool = True,
+                   fk_schema: str | None = None, fk_table: str | None = None, fk_column: str | None = None) -> None:
+        '''Adds a column to the catalog, creating the schema and table if they do not exist.'''
+
+        self[schema_name][table_name].add_column(name=column_name,
+                                                 column_type=column_type, numeric_precision=numeric_precision, numeric_scale=numeric_scale,
+                                                 is_nullable=is_nullable,
+                                                 fk_schema=fk_schema, fk_table=fk_table, fk_column=fk_column)
+        
+    @property
+    def schema_names(self) -> set[str]:
+        '''Returns all schema names in the catalog.'''
+        return set(self._schemas.keys())
+
+    @property
+    def table_names(self) -> set[str]:
+        '''Returns all table names in the catalog, regardless of schema.'''
+
+        result = set()
+        for schema in self._schemas.values():
+            result.update(schema.table_names)
+        return result
+
+    def copy(self) -> Self:
+        '''Creates a deep copy of the catalog.'''
+        return deepcopy(self)
+    
+    def __repr__(self) -> str:
+        schemas = [schema.__repr__(1) for schema in self._schemas.values()]
+
+        result = 'Catalog('
+        for schema in schemas:
+            result += '\n' + schema
+        result += '\n)'
+
+        return result
+
+    
+    def to_dict(self) -> dict:
+        return {
+            'version': 1,
+            'schemas': {name: sch.to_dict() for name, sch in self._schemas.items()},
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> 'Catalog':
+        cat = cls()
+        for _, sch_data in (data.get('schemas') or {}).items():
+            sch = Schema.from_dict(sch_data)
+            cat._schemas[sch.name] = sch
+        return cat
+
+    # String-based JSON (handy for DB/blob storage)
+    def to_json(self, *, indent: int | None = 2) -> str:
+        return json.dumps(self.to_dict(), indent=indent)
+
+    @classmethod
+    def from_json(cls, s: str) -> 'Catalog':
+        return cls.from_dict(json.loads(s))
+
+    # Convenience file helpers
+    def save_json(self, path: str, *, indent: int | None = 2) -> None:
+        with open(path, 'w', encoding='utf-8') as f:
+            json.dump(self.to_dict(), f, indent=indent)
+
+    @classmethod
+    def load_json(cls, path: str) -> 'Catalog':
+        with open(path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return cls.from_dict(data)

sql_error_categorizer/catalog/queries.py

@@ -0,0 +1,60 @@
+def UNIQUE_COLUMNS(schema_name: str = '%') -> str:
+    return f'''
+    SELECT
+        kcu.table_schema AS schema_name,
+        kcu.table_name,
+        tc.constraint_type,
+        array_agg(kcu.column_name ORDER BY kcu.ordinal_position) AS columns
+    FROM information_schema.table_constraints tc
+    JOIN information_schema.key_column_usage kcu
+    ON tc.constraint_name = kcu.constraint_name
+    AND tc.constraint_schema = kcu.constraint_schema
+    WHERE tc.constraint_type IN ('UNIQUE', 'PRIMARY KEY')
+    AND kcu.table_schema LIKE '{schema_name}'
+    GROUP BY
+        kcu.table_schema,
+        kcu.table_name,
+        kcu.constraint_name,
+        tc.constraint_type;
+'''
+
+def COLUMNS(schema_name: str = '%') -> str:
+    return f'''
+    SELECT
+        cols.table_schema AS schema_name,
+        cols.table_name,
+        cols.column_name,
+        cols.data_type AS column_type,
+        cols.numeric_precision,
+        cols.numeric_scale,
+        (cols.is_nullable = 'YES') AS is_nullable,
+        fk.foreign_table_schema AS foreign_key_schema,
+        fk.foreign_table_name AS foreign_key_table,
+        fk.foreign_column_name AS foreign_key_column
+    FROM information_schema.columns AS cols
+
+    -- Foreign Key
+    LEFT JOIN (
+        SELECT
+            kcu.table_schema,
+            kcu.table_name,
+            kcu.column_name,
+            ccu.table_schema AS foreign_table_schema,
+            ccu.table_name   AS foreign_table_name,
+            ccu.column_name  AS foreign_column_name
+        FROM information_schema.table_constraints AS tc
+            JOIN information_schema.key_column_usage AS kcu
+            ON tc.constraint_name = kcu.constraint_name
+            AND tc.constraint_schema = kcu.constraint_schema
+            AND tc.table_schema = kcu.table_schema
+            AND tc.table_name = kcu.table_name
+            JOIN information_schema.constraint_column_usage AS ccu
+            ON tc.constraint_name = ccu.constraint_name
+            AND tc.constraint_schema = ccu.constraint_schema
+        WHERE tc.constraint_type = 'FOREIGN KEY'
+    ) fk ON fk.table_schema = cols.table_schema
+        AND fk.table_name = cols.table_name
+        AND fk.column_name = cols.column_name
+
+    WHERE cols.table_schema LIKE '{schema_name}'
+'''

sql_error_categorizer/detectors/__init__.py

@@ -0,0 +1,88 @@
+from .. import catalog
+from ..query import Query
+from ..sql_errors import SqlErrors
+from .base import BaseDetector, DetectedError
+from .syntax import SyntaxErrorDetector
+from .semantic import SemanticErrorDetector
+from .logical import LogicalErrorDetector
+from .complications import ComplicationDetector
+
+class Detector:
+    def __init__(self,
+                 query: str,
+                 *,
+                 search_path: str = 'public',
+                 solution_search_path: str = 'public',
+                 solutions: list[str] = [],
+                 catalog: catalog.Catalog = catalog.Catalog(),
+                 detectors: list[type[BaseDetector]] = [],
+                 debug: bool = False):
+        
+        # Context data: they don't need to be parsed again if the query changes
+        self.search_path = search_path
+        self.solution_search_path = solution_search_path
+        self.catalog = catalog
+        self.solutions = [Query(sol, catalog=self.catalog, search_path=self.solution_search_path) for sol in solutions]
+        self.detectors: list[BaseDetector] = []
+        self.debug = debug
+
+        self.set_query(query)
+
+        # NOTE: Add detectors after setting the query to ensure they are correctly initialized
+        for detector_cls in detectors:
+            self.add_detector(detector_cls)
+
+    def set_query(self, query: str, reason: str | None = None) -> None:
+        '''Set a new query, re-parse it, and update all detectors. Doesn't affect detected errors.'''
+        
+        if self.debug:
+            print('=' * 20)
+            if reason:
+                print(f'Updating query ({reason}):\n{query}')
+            else:
+                print(f'Updating query:\n{query}')
+            print('=' * 20)
+
+        self.query = Query(query, catalog=self.catalog, search_path=self.search_path)
+
+        # Update all detectors with the new query and parse results
+        for detector in self.detectors:
+            detector.query = self.query
+            detector.update_query = lambda new_query, reason=None: self.set_query(new_query, reason)
+
+    def add_detector(self, detector_cls: type[BaseDetector]) -> None:
+        '''Add a detector instance to the list of detectors'''
+
+        # Make copies to avoid possible modifications during detection
+        # TODO: check if it's needed
+        detector = detector_cls(
+            query=self.query,
+            solutions=self.solutions,
+            update_query=lambda new_query, reason=None: self.set_query(new_query, reason),
+        )
+
+        self.detectors.append(detector)
+
+    def run(self) -> list[DetectedError]:
+        '''
+        Run all detectors and return a list of detected errors.
+        This function can return duplicate errors, as well as additional information on the detected errors.
+        '''
+
+        if self.debug:
+            print('===== Query =====')
+            print(self.query.sql)
+
+        results: list[DetectedError] = []
+
+        for detector in self.detectors:
+            errors = detector.run()
+
+            if self.debug:
+                print(f'===== Detected errors from {detector.__class__.__name__} =====')
+                for error in errors:
+                    print(error)
+
+            results.extend(errors)
+
+        return results

sql_error_categorizer/detectors/base.py

@@ -0,0 +1,39 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from ..sql_errors import SqlErrors
+from ..query import Query
+from ..catalog import Catalog
+
+@dataclass(repr=False)
+class DetectedError:
+    error: SqlErrors
+    data: tuple[Any, ...] = field(default_factory=tuple)
+
+    def __repr__(self):
+        return f"DetectedError({self.error.value} - {self.error.name}: {self.data})"
+
+    def __str__(self) -> str:
+        if self.data:
+            return f'[{self.error.value:3}] {self.error.name}: {self.data}'
+        return f'[{self.error.value:3}] {self.error.name}'
+    
+    def __hash__(self) -> int:
+        return hash((self.error, self.data))
+
+class BaseDetector(ABC):
+    def __init__(self, *,
+                 query: Query,
+                 solutions: list[Query] = [],
+                 update_query: Callable[[str, str | None], None],
+        ):        
+        self.query = query
+        self.solutions = solutions
+        self.update_query = update_query
+
+    @abstractmethod
+    def run(self) -> list[DetectedError]:
+        '''Run the detector and return a list of detected errors with their descriptions'''
+        return []
+