velocity-python 0.0.35__py3-none-any.whl → 0.0.65__py3-none-any.whl

velocity/db/servers/{postgres.py → postgres/sql.py}

@@ -1,13 +1,20 @@
-import psycopg2
 import re
-import os
 import hashlib
-import decimal
-import datetime
-from ..core import exceptions
-from ..core import engine
-from ..core.table import Query
-from .postgres_reserved import reserved_words
+import sqlparse
+
+
+from velocity.db.core import exceptions
+
+from .reserved import reserved_words
+from .types import TYPES
+from .operators import OPERATORS
+from ..tablehelper import TableHelper
+from collections.abc import Mapping, Sequence
+
+
+TableHelper.reserved = reserved_words
+TableHelper.operators = OPERATORS
+
 
 system_fields = [
     "sys_id",
@@ -19,162 +26,8 @@ system_fields = [
     "description",
 ]
 
-default_config = {
-    "database": os.environ["DBDatabase"],
-    "host": os.environ["DBHost"],
-    "port": os.environ["DBPort"],
-    "user": os.environ["DBUser"],
-    "password": os.environ["DBPassword"],
-}
-
-def initialize(config=None, **kwargs):
-    if not config:
-        config = default_config.copy()
-    config.update(kwargs)
-    return engine.Engine(psycopg2, config, SQL)
-
-def make_where(where, sql, vals, is_join=False):
-    if not where:
-        return
-    sql.append("WHERE")
-    if isinstance(where, str):
-        sql.append(where)
-        return
-    if isinstance(where, dict):
-        where = list(where.items())
-    if not isinstance(where, list):
-        raise Exception("Parameter `where` is not a valid datatype.")
-    alias = "A"
-    if is_join and isinstance(is_join, str):
-        alias = is_join
-    connect = ""
-    for key, val in where:
-        if connect:
-            sql.append(connect)
-        if is_join:
-            if "." not in key:
-                key = f"{alias}.{quote(key.lower())}"
-        if val is None:
-            if "!" in key:
-                key = key.replace("!", "")
-                sql.append(f"{key} is not NULL")
-            else:
-                sql.append(f"{key} is NULL")
-        elif isinstance(val, (list, tuple)) and "><" not in key:
-            if "!" in key:
-                key = key.replace("!", "")
-                sql.append(f"{key} not in %s")
-                vals.append(tuple(val))
-            else:
-                sql.append(f"{key} in %s")
-                vals.append(tuple(val))
-        elif isinstance(val, Query):
-            sql.append(f"{key} in ({val})")
-        else:
-            case = None
-            if "<>" in key:
-                key = key.replace("<>", "")
-                op = "<>"
-            elif "!=" in key:
-                key = key.replace("!=", "")
-                op = "<>"
-            elif "!><" in key:
-                key = key.replace("!><", "")
-                op = "not between"
-            elif "><" in key:
-                key = key.replace("><", "")
-                op = "between"
-            elif "!%" in key:
-                key = key.replace("!%", "")
-                op = "not like"
-            elif "%%" in key:
-                key = key.replace("%%", "")
-                op = "%"
-            elif "%>" in key:
-                key = key.replace("%>", "")
-                op = "%>"
-            elif "<%" in key:
-                key = key.replace("<%", "")
-                op = "<%"
-            elif "==" in key:
-                key = key.replace("==", "")
-                op = "="
-            elif "<=" in key:
-                key = key.replace("<=", "")
-                op = "<="
-            elif ">=" in key:
-                key = key.replace(">=", "")
-                op = ">="
-            elif "<" in key:
-                key = key.replace("<", "")
-                op = "<"
-            elif ">" in key:
-                key = key.replace(">", "")
-                op = ">"
-            elif "%" in key:
-                key = key.replace("%", "")
-                op = "ilike"
-            elif "!~*" in key:
-                key = key.replace("!~*", "")
-                op = "!~*"
-            elif "~*" in key:
-                key = key.replace("~*", "")
-                op = "~*"
-            elif "!~" in key:
-                key = key.replace("!~", "")
-                op = "!~"
-            elif "~" in key:
-                key = key.replace("~", "")
-                op = "~"
-            elif "!" in key:
-                key = key.replace("!", "")
-                op = "<>"
-            elif "=" in key:
-                key = key.replace("=", "")
-                op = "="
-            else:
-                op = "="
-            if "#" in key:
-                key = key.replace("#", "")
-                op = "="
-                case = "lower"
-            if isinstance(val, str) and val[:2] == "@@" and val[2:]:
-                sql.append(f"{key} {op} {val[2:]}")
-            elif op in ["between", "not between"]:
-                sql.append(f"{key} {op} %s and %s")
-                vals.extend(val)
-            else:
-                if case:
-                    sql.append(f"{case}({key}) {op} {case}(%s)")
-                else:
-                    sql.append(f"{key} {op} %s")
-                vals.append(val)
-        connect = "AND"
-
-def quote(data):
-    if isinstance(data, list):
-        new = []
-        for item in data:
-            if "@@" in item:
-                new.append(item[2:])
-            else:
-                new.append(quote(item))
-        return new
-    else:
-        parts = data.split(".")
-        new = []
-        for part in parts:
-            if '"' in part:
-                new.append(part)
-            elif part.upper() in reserved_words:
-                new.append(f'"{part}"')
-            elif re.findall("[/]", part):
-                new.append(f'"{part}"')
-            else:
-                new.append(part)
-        return ".".join(new)
 
-class SQL(object):
+class SQL:
     server = "PostGreSQL"
     type_column_identifier = "data_type"
     is_nullable = "is_nullable"
@@ -188,7 +41,7 @@ class SQL(object):
     ColumnMissingErrorCodes = ["42703"]
     ForeignKeyMissingErrorCodes = ["42704"]
 
-    ConnectionErrorCodes = ["08001", "08S01","57P03", "08006", "53300"]
+    ConnectionErrorCodes = ["08001", "08S01", "57P03", "08006", "53300"]
     DuplicateKeyErrorCodes = []  # Handled in regex check.
     RetryTransactionCodes = []
     TruncationErrorCodes = ["22001"]
@@ -196,6 +49,388 @@ class SQL(object):
     DatabaseObjectExistsErrorCodes = ["42710", "42P07", "42P04"]
     DataIntegrityErrorCodes = ["23503"]
 
+    @classmethod
+    def get_error(self, e):
+        error_code = getattr(e, "pgcode", None)
+        error_mesg = getattr(e, "pgerror", None)
+        return error_code, error_mesg
+
+    types = TYPES
+
+    @classmethod
+    def select(
+        cls,
+        tx,
+        columns=None,
+        table=None,
+        where=None,
+        orderby=None,
+        groupby=None,
+        having=None,
+        start=None,
+        qty=None,
+        lock=None,
+        skip_locked=None,
+    ):
+
+        if not table:
+            raise ValueError("Table name is required.")
+
+        sql_parts = {
+            "SELECT": [],
+            "FROM": [],
+            "WHERE": [],
+            "GROUP BY": [],
+            "HAVING": [],
+            "ORDER BY": [],
+        }
+
+        sql = []
+        vals = []
+
+        # Assume these helpers and functions exist externally
+        th = TableHelper(tx, table)
+
+        # Handle columns and DISTINCT before aliasing
+        if columns is None:
+            # No columns specified - select all
+            columns = ["*"]
+        elif isinstance(columns, str):
+            columns = th.split_columns(columns)
+
+        if not isinstance(columns, Sequence):
+            raise Exception(
+                f"variable `columns` must be a sequence, but {type(columns)} was found"
+            )
+
+        columns = [c.strip() for c in columns]  # Preserve original case
+        distinct = False
+
+        if any(
+            "distinct" in c.lower() for c in columns
+        ):  # Check if "distinct" exists in any entry
+            distinct = True
+            columns = [
+                c.replace("distinct", "", 1).strip() if "distinct" in c.lower() else c
+                for c in columns
+            ]
+
+        processed_columns = []
+        for col in columns:
+            processed_columns.append(
+                th.resolve_references(
+                    col, options={"alias_column": True, "alias_table": True}
+                )
+            )
+
+        columns = processed_columns
+
+        # Handle WHERE conditions
+        if isinstance(where, Mapping):
+            new_where = []
+            for key, val in where.items():
+                new_where.append(th.make_predicate(key, val))
+            where = new_where
+
+        new_orderby = []
+        if isinstance(orderby, str):
+            orderby = th.split_columns(orderby)
+        # Handle orderby references
+        if isinstance(orderby, (Sequence)):
+            for column in orderby:
+                if " " in column:
+                    col_name, direction = column.split(" ", 1)
+                    col_name = th.resolve_references(
+                        col_name, options={"alias_only": True}
+                    )
+                    new_orderby.append(f"{col_name} {direction}")
+                else:
+                    new_orderby.append(
+                        th.resolve_references(
+                            column.strip(), options={"alias_only": True}
+                        )
+                    )
+
+        if isinstance(orderby, Mapping):
+            for key, val in orderby.items():
+                parsed_key = th.resolve_references(key, options={"alias_only": True})
+                new_orderby.append(f"{parsed_key} {val}")
+        orderby = new_orderby
+
+        # Handle groupby
+        if isinstance(groupby, str):
+            groupby = th.split_columns(groupby)
+        if isinstance(groupby, (Sequence)):
+            new_groupby = []
+            for gcol in groupby:
+                new_groupby.append(
+                    th.resolve_references(gcol, options={"alias_only": True})
+                )
+            groupby = new_groupby
+
+        # Handle having
+        if isinstance(having, Mapping):
+            new_having = []
+            for key, val in having.items():
+                new_having.append(th.make_predicate(key, val))
+            having = new_having
+
+        # SELECT clause
+        # columns is a list/tuple of already processed references
+        sql_parts["SELECT"].extend(columns)
+        alias = th.get_table_alias("current_table")
+        if not alias:
+            raise ValueError("Main table alias resolution failed.")
+
+        # FROM clause
+        if th.foreign_keys:
+            sql_parts["FROM"].append(f"{th.quote(table)} AS {th.quote(alias)}")
+            # Handle joins
+            done = []
+            for key, ref_info in th.foreign_keys.items():
+                ref_table = ref_info["ref_table"]
+                if ref_table in done:
+                    continue
+                done.append(ref_table)
+                if not all(
+                    k in ref_info
+                    for k in ("alias", "local_column", "ref_table", "ref_column")
+                ):
+                    raise ValueError(f"Invalid table alias info for {ref_table}.")
+                sql_parts["FROM"].append(
+                    f"LEFT JOIN {th.quote(ref_table)} AS {th.quote(ref_info['alias'])} "
+                    f"ON {th.quote(alias)}.{th.quote(ref_info['local_column'])} = {th.quote(ref_info['alias'])}.{th.quote(ref_info['ref_column'])}"
+                )
+        else:
+            sql_parts["FROM"].append(th.quote(table))
+
+        # WHERE
+        if where:
+            if isinstance(where, str):
+                sql_parts["WHERE"].append(where)
+            else:
+                for pred, val in where:
+                    sql_parts["WHERE"].append(pred)
+                    if val is None:
+                        pass
+                    elif isinstance(val, tuple):
+                        vals.extend(val)
+                    else:
+                        vals.append(val)
+
+        # GROUP BY
+        if groupby:
+            sql_parts["GROUP BY"].append(",".join(groupby))
+
+        # HAVING
+        if having:
+            if isinstance(having, str):
+                sql_parts["HAVING"].append(having)
+            else:
+                for pred, val in having:
+                    sql_parts["HAVING"].append(pred)
+                    if val is None:
+                        pass
+                    elif isinstance(val, tuple):
+                        vals.extend(val)
+                    else:
+                        vals.append(val)
+
+        # ORDER BY
+        if orderby:
+            sql_parts["ORDER BY"].append(",".join(orderby))
+
+        # Construct final SQL
+        if sql_parts["SELECT"]:
+            sql.append("SELECT")
+            if distinct:
+                sql.append("DISTINCT")
+            sql.append(", ".join(sql_parts["SELECT"]))
+
+        if sql_parts["FROM"]:
+            sql.append("FROM")
+            sql.append(" ".join(sql_parts["FROM"]))
+
+        if sql_parts["WHERE"]:
+            sql.append("WHERE " + " AND ".join(sql_parts["WHERE"]))
+
+        if sql_parts["GROUP BY"]:
+            sql.append("GROUP BY " + " ".join(sql_parts["GROUP BY"]))
+
+        if sql_parts["HAVING"]:
+            sql.append("HAVING " + " AND ".join(sql_parts["HAVING"]))
+
+        if sql_parts["ORDER BY"]:
+            sql.append("ORDER BY " + " ".join(sql_parts["ORDER BY"]))
+
+        # OFFSET/FETCH
+        if start is not None:
+            if not isinstance(start, int):
+                raise ValueError("Start (OFFSET) must be an integer.")
+            sql.append(f"OFFSET {start} ROWS")
+
+        if qty is not None:
+            if not isinstance(qty, int):
+                raise ValueError("Qty (FETCH) must be an integer.")
+            sql.append(f"FETCH NEXT {qty} ROWS ONLY")
+
+        # FOR UPDATE and SKIP LOCKED
+        if lock or skip_locked:
+            sql.append("FOR UPDATE")
+        if skip_locked:
+            sql.append("SKIP LOCKED")
+
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple(vals)
+
+    @classmethod
+    def update(cls, tx, table, data, where=None, pk=None, excluded=False):
+        if not table:
+            raise ValueError("Table name is required.")
+        if not pk and not where:
+            raise ValueError("Where clause (where) or primary key (pk) is required.")
+        if not isinstance(data, Mapping) or not data:
+            raise ValueError("data must be a non-empty mapping of column-value pairs.")
+
+        th = TableHelper(tx, table)
+
+        set_clauses = []
+        vals = []
+
+        if pk:
+            if where:
+                where.update(pk)
+            else:
+                where = pk
+
+        # Handle data columns (SET clause)
+        for col, val in data.items():
+            col = th.resolve_references(
+                col, options={"alias_column": False, "alias_table": False}
+            )
+
+            # Normal column
+            if excluded:
+                set_clauses.append(f"{col} = EXCLUDED.{col}")
+            else:
+                set_clauses.append(f"{col} = %s")
+                vals.append(val)
+
+        # Extract the final where conditions and values
+        where_clauses = []
+        if not excluded:
+            # First handle user-provided WHERE conditions
+            if isinstance(where, Mapping):
+                for key, val in where.items():
+                    col, value = th.make_predicate(key, val)
+                    where_clauses.append(col)
+                    if isinstance(value, tuple):
+                        vals.extend(value)
+                    else:
+                        vals.append(value)
+
+        # Final assembly of SQL
+        sql = []
+        sql.append("UPDATE")
+        if not excluded:
+            if th.foreign_keys:
+                sql.append(
+                    f"{th.quote(table)} AS {th.quote(th.get_table_alias('current_table'))}"
+                )
+            else:
+                sql.append(TableHelper.quote(table))
+        sql.append("SET " + ", ".join(set_clauses))
+        if not excluded:
+            if th.foreign_keys:
+                for key, ref_info in th.foreign_keys.items():
+                    ref_table = ref_info["ref_table"]
+                    sql.append(
+                        f"LEFT JOIN {th.quote(ref_table)} AS {th.quote(ref_info['alias'])} "
+                    )
+                    where_clauses.append(
+                        f"{th.quote(th.get_table_alias('current_table'))}.{th.quote(ref_info['local_column'])} = {th.quote(ref_info['alias'])}.{th.quote(ref_info['ref_column'])}"
+                    )
+
+        if not excluded:
+            if where_clauses:
+                sql.append("WHERE " + " AND ".join(where_clauses))
+            else:
+                # Without a WHERE, this updates all rows.
+                # If this is not desired, raise an error.
+                if not excluded:
+                    raise ValueError(
+                        "No WHERE clause could be constructed. Update would affect all rows."
+                    )
+
+        # Final assembled query
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple(vals)
+
+    @classmethod
+    def insert(cls, table, data):
+        keys = []
+        vals = []
+        args = []
+        for key, val in data.items():
+            keys.append(TableHelper.quote(key.lower()))
+            if isinstance(val, str) and len(val) > 2 and val[:2] == "@@" and val[2:]:
+                vals.append(val[2:])
+            else:
+                vals.append("%s")
+                args.append(val)
+
+        sql = ["INSERT INTO"]
+        sql.append(TableHelper.quote(table))
+        sql.append("(")
+        sql.append(",".join(keys))
+        sql.append(")")
+        sql.append("VALUES")
+        sql.append("(")
+        sql.append(",".join(vals))
+        sql.append(")")
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple(args)
+
+    @classmethod
+    def merge(cls, tx, table, data, pk, on_conflict_do_nothing, on_conflict_update):
+        if pk is None:
+            pkeys = tx.table(table).primary_keys()
+            if not pkeys:
+                raise ValueError("Primary key required for merge.")
+            # If there are multiple primary keys, we need to use all of them
+            if len(pkeys) > 1:
+                pk = {pk: data[pk] for pk in pkeys}
+            else:
+                pk = {pkeys[0]: data[pkeys[0]]}
+            # remove primary key from data
+            data = {k: v for k, v in data.items() if k not in pk}
+
+        full_data = {}
+        full_data.update(data)
+        full_data.update(pk)
+
+        sql, vals = cls.insert(table, full_data)
+        sql = [sql]
+        vals = list(vals)
+        if on_conflict_do_nothing != on_conflict_update:
+            sql.append("ON CONFLICT")
+            sql.append("(")
+            sql.append(",".join(pk.keys()))
+            sql.append(")")
+            sql.append("DO")
+            if on_conflict_do_nothing:
+                sql.append("NOTHING")
+            elif on_conflict_update:
+                sql2, vals2 = cls.update(tx, table, data, pk, excluded=True)
+                sql.append(sql2)
+                vals.extend(vals2)
+        else:
+            raise Exception(
+                "Update on conflict must have one and only one option to complete on conflict."
+            )
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple(vals)
+
     @classmethod
     def version(cls):
         return "select version()", tuple()
@@ -250,143 +485,6 @@ class SQL(object):
                 tuple(),
             )
 
-    @classmethod
-    def __has_pointer(cls, columns):
-        if isinstance(columns, str):
-            columns = columns.split(",")
-        if isinstance(columns, list):
-            for column in columns:
-                if "@@" in column:
-                    continue
-                if ">" in column:
-                    return True
-        return False
-
-    @classmethod
-    def select(
-        cls,
-        columns=None,
-        table=None,
-        where=None,
-        orderby=None,
-        groupby=None,
-        having=None,
-        start=None,
-        qty=None,
-        tbl=None,
-        lock=None,
-        skip_locked=None,
-    ):
-        if not table:
-            raise Exception("Table name required")
-        is_join = False
-
-        if isinstance(columns, str) and "distinct" in columns.lower():
-            sql = [
-                "SELECT",
-                columns,
-                "FROM",
-                quote(table),
-            ]
-        elif cls.__has_pointer(columns):
-            is_join = True
-            if isinstance(columns, str):
-                columns = columns.split(",")
-            letter = 65
-            tables = {table: chr(letter)}
-            letter += 1
-            __select = []
-            __from = [f"{quote(table)} AS {tables.get(table)}"]
-            __left_join = []
-
-            for column in columns:
-                if "@@" in column:
-                    __select.append(column[2:])
-                elif ">" in column:
-                    parts = column.split(">")
-                    foreign = tbl.foreign_key_info(parts[0])
-                    if not foreign:
-                        raise exceptions.DbApplicationError("Foreign key not defined")
-                    ref_table = foreign["referenced_table_name"]
-                    ref_schema = foreign["referenced_table_schema"]
-                    ref_column = foreign["referenced_column_name"]
-                    lookup = f"{ref_table}:{parts[0]}"
-                    if lookup in tables:
-                        __select.append(
-                            f'{tables.get(lookup)}."{parts[1]}" as "{'_'.join(parts)}"'
-                        )
-                    else:
-                        tables[lookup] = chr(letter)
-                        letter += 1
-                        __select.append(
-                            f'{tables.get(lookup)}."{parts[1]}" as "{'_'.join(parts)}"'
-                        )
-                        __left_join.append(
-                            f'LEFT OUTER JOIN "{ref_schema}"."{ref_table}" AS {tables.get(lookup)}'
-                        )
-                        __left_join.append(
-                            f'ON {tables.get(table)}."{parts[0]}" = {tables.get(lookup)}."{ref_column}"'
-                        )
-                    if orderby and column in orderby:
-                        orderby = orderby.replace(
-                            column, f"{tables.get(lookup)}.{parts[1]}"
-                        )
-
-                else:
-                    if "(" in column:
-                        __select.append(column)
-                    else:
-                        __select.append(f"{tables.get(table)}.{column}")
-            sql = ["SELECT"]
-            sql.append(",".join(__select))
-            sql.append("FROM")
-            sql.extend(__from)
-            sql.extend(__left_join)
-        else:
-            if columns:
-                if isinstance(columns, str):
-                    columns = columns.split(",")
-                if isinstance(columns, list):
-                    columns = quote(columns)
-                    columns = ",".join(columns)
-            else:
-                columns = "*"
-            sql = [
-                "SELECT",
-                columns,
-                "FROM",
-                quote(table),
-            ]
-        vals = []
-        make_where(where, sql, vals, is_join)
-        if groupby:
-            sql.append("GROUP BY")
-            if isinstance(groupby, (list, tuple)):
-                groupby = ",".join(groupby)
-            sql.append(groupby)
-        if having:
-            sql.append("HAVING")
-            if isinstance(having, (list, tuple)):
-                having = ",".join(having)
-            sql.append(having)
-        if orderby:
-            sql.append("ORDER BY")
-            if isinstance(orderby, (list, tuple)):
-                orderby = ",".join(orderby)
-            sql.append(orderby)
-        if start and qty:
-            sql.append(f"OFFSET {start} ROWS FETCH NEXT {qty} ROWS ONLY")
-        elif start:
-            sql.append(f"OFFSET {start} ROWS")
-        elif qty:
-            sql.append(f"FETCH NEXT {qty} ROWS ONLY")
-        if lock or skip_locked:
-            sql.append("FOR UPDATE")
-        if skip_locked:
-            sql.append("SKIP LOCKED")
-        sql = " ".join(sql)
-        return sql, tuple(vals)
-
     @classmethod
     def create_database(cls, name):
         return f"create database {name}", tuple()
@@ -469,24 +567,26 @@ class SQL(object):
             if key in system_fields:
                 continue
             sql.append(
-                f"ALTER TABLE {quote(fqtn)} ADD COLUMN {quote(key)} {cls.get_type(val)};"
+                f"ALTER TABLE {TableHelper.quote(fqtn)} ADD COLUMN {TableHelper.quote(key)} {TYPES.get_type(val)};"
             )
-        return "\n\t".join(sql), tuple()
+
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple()
 
     @classmethod
     def drop_table(cls, name):
-        return f"drop table if exists {quote(name)} cascade;", tuple()
+        return f"drop table if exists {TableHelper.quote(name)} cascade;", tuple()
 
     @classmethod
     def drop_column(cls, table, name, cascade=True):
         if cascade:
             return (
-                f"ALTER TABLE {quote(table)} DROP COLUMN {quote(name)} CASCADE",
+                f"ALTER TABLE {TableHelper.quote(table)} DROP COLUMN {TableHelper.quote(name)} CASCADE",
                 tuple(),
             )
         else:
             return (
-                f"ALTER TABLE {quote(table)} DROP COLUMN {quote(name)}",
+                f"ALTER TABLE {TableHelper.quote(table)} DROP COLUMN {TableHelper.quote(name)}",
                 tuple(),
             )
 
@@ -616,8 +716,17 @@ class SQL(object):
             where["LOWER(KCU1.TABLE_NAME)"] = table.lower()
         if column:
             where["LOWER(KCU1.COLUMN_NAME)"] = column.lower()
-        make_where(where, sql, vals)
-        return " ".join(sql), tuple(vals)
+        sql.append("WHERE")
+        connect = ""
+        for key, val in where.items():
+            if connect:
+                sql.append(connect)
+            sql.append(f"{key} = %s")
+            vals.append(val)
+            connect = "AND"
+
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple(vals)
 
     @classmethod
     def create_foreign_key(
@@ -637,7 +746,7 @@ class SQL(object):
             m.update(" ".join(key_to_columns).encode("utf-8"))
             name = f"FK_{m.hexdigest()}"
         sql = f"ALTER TABLE {table} ADD CONSTRAINT {name} FOREIGN KEY ({','.join(columns)}) REFERENCES {key_to_table} ({','.join(key_to_columns)});"
-
+        sql = sqlparse.format(sql, reindent=True, keyword_case="upper")
         return sql, tuple()
 
     @classmethod
@@ -669,6 +778,7 @@ class SQL(object):
     @classmethod
     def create_index(
         cls,
+        tx,
         table=None,
         columns=None,
         unique=False,
@@ -677,7 +787,6 @@ class SQL(object):
         name=None,
         schema=None,
         trigram=None,
-        tbl=None,
         lower=None,
     ):
         """
@@ -688,14 +797,14 @@ class SQL(object):
         if "." not in table and schema:
             table = f"{schema}.{table}"
         if isinstance(columns, (list, set)):
-            columns = ",".join([quote(c.lower()) for c in columns])
+            columns = ",".join([TableHelper.quote(c.lower()) for c in columns])
         else:
-            columns = quote(columns)
+            columns = TableHelper.quote(columns)
         sql = ["CREATE"]
         if unique:
             sql.append("UNIQUE")
         sql.append("INDEX")
-        tablename = quote(table)
+        tablename = TableHelper.quote(table)
         if not name:
             name = re.sub(
                 r"\([^)]*\)",
@@ -703,54 +812,58 @@ class SQL(object):
                 columns.replace(" ", "").replace(",", "_").replace('"', ""),
             )
         if trigram:
-            sql.append(
-                f"IDX__TRGM_{table.replace('.', '_')}_{trigram.upper()}__{name}"
-            )
+            sql.append(f"IDX__TRGM_{table.replace('.', '_')}_{trigram}__{name}".upper())
         else:
-            sql.append(f"IDX__{table.replace('.', '_')}__{name}")
+            sql.append(f"IDX__{table.replace('.', '_')}__{name}".upper())
         sql.append("ON")
-        sql.append(quote(tablename))
+        sql.append(TableHelper.quote(tablename))
 
         if trigram:
             sql.append("USING")
             sql.append(trigram)
         sql.append("(")
-        if tbl:
-            join = ""
-            for column_name in columns.split(","):
-                column_name = column_name.replace('"', "")
-                if join:
-                    sql.append(join)
-                column = tbl.column(column_name)
-                if column.py_type == str:
-                    if lower:
-                        sql.append(f"lower({quote(column_name)})")
-                    else:
-                        sql.append(quote(column_name))
+        join = ""
+        for column_name in columns.split(","):
+            column_name = column_name.replace('"', "")
+            if join:
+                sql.append(join)
+            column = tx.table(table).column(column_name)
+            print(column)
+            if not column.exists():
+                raise Exception(
+                    f"Column {column_name} does not exist in table {table}."
+                )
+            if column.py_type == str:
+                if lower:
+                    sql.append(f"lower({TableHelper.quote(column_name)})")
                 else:
-                    sql.append(quote(column_name))
-                join = ","
-        else:
-            sql.append(columns)
+                    sql.append(TableHelper.quote(column_name))
+            else:
+                sql.append(TableHelper.quote(column_name))
+            join = ","
+
         if trigram:
             sql.append(f"{trigram.lower()}_trgm_ops")
         sql.append(")")
         vals = []
+        s, v = TableHelper(tx, table).make_where(where)
+        sql.append(s)
+        vals.extend(v)
 
-        make_where(where, sql, vals)
-        return " ".join(sql), tuple(vals)
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple(vals)
 
     @classmethod
     def drop_index(cls, table=None, columns=None, name=None, schema=None, trigram=None):
         if "." not in table and schema:
             table = f"{schema}.{table}"
         if isinstance(columns, (list, set)):
-            columns = ",".join([quote(c.lower()) for c in columns])
+            columns = ",".join([TableHelper.quote(c.lower()) for c in columns])
         else:
-            columns = quote(columns)
+            columns = TableHelper.quote(columns)
         sql = ["DROP"]
         sql.append("INDEX IF EXISTS")
-        tablename = quote(table)
+        tablename = TableHelper.quote(table)
         if not name:
             name = re.sub(
                 r"\([^)]*\)",
@@ -758,209 +871,12 @@ class SQL(object):
                 columns.replace(" ", "").replace(",", "_").replace('"', ""),
             )
         if trigram:
-            sql.append(
-                f"IDX__TRGM_{table.replace('.', '_')}_{trigram.upper()}__{name}"
-            )
+            sql.append(f"IDX__TRGM_{table.replace('.', '_')}_{trigram.upper()}__{name}")
         else:
             sql.append(f"IDX__{table.replace('.', '_')}__{name}")
-        return " ".join(sql), tuple()
 
-    @classmethod
-    def merge(cls, table, data, pk, on_conflict_do_nothing, on_conflict_update):
-        d = {}
-        d.update(data)
-        d.update(pk)
-        sql, vals = cls.insert(table, d)
-        sql = [sql]
-        vals = list(vals)
-        if on_conflict_do_nothing != on_conflict_update:
-            sql.append("ON CONFLICT")
-            sql.append("(")
-            sql.append(",".join(pk.keys()))
-            sql.append(")")
-            sql.append("DO")
-            if on_conflict_do_nothing:
-                sql.append("NOTHING")
-            elif on_conflict_update:
-                sql2, vals2 = cls.update(table, data, pk, excluded=True)
-                sql.append(sql2)
-                vals.extend(vals2)
-        else:
-            raise Exception(
-                "Update on conflict must have one and only one option to complete on conflict."
-            )
-        return " ".join(sql), tuple(vals)
-
-    @classmethod
-    def insert(cls, table, data):
-        keys = []
-        vals = []
-        args = []
-        for key, val in data.items():
-            keys.append(quote(key.lower()))
-            if isinstance(val, str) and len(val) > 2 and val[:2] == "@@" and val[2:]:
-                vals.append(val[2:])
-            else:
-                vals.append("%s")
-                args.append(val)
-
-        sql = ["INSERT INTO"]
-        sql.append(quote(table))
-        sql.append("(")
-        sql.append(",".join(keys))
-        sql.append(")")
-        sql.append("VALUES")
-        sql.append("(")
-        sql.append(",".join(vals))
-        sql.append(")")
-        sql = " ".join(sql)
-        return sql, tuple(args)
-
-    @classmethod
-    def update(
-        cls,
-        table,
-        data,
-        pk,
-        left_join=None,
-        inner_join=None,
-        outer_join=None,
-        excluded=False,
-    ):
-        alias = "A"
-        if " " in table:
-            alias, table = table.split(" ")
-        is_join = bool(left_join or inner_join or outer_join)
-        sql = ["UPDATE"]
-        if not excluded:
-            sql.append(quote(table))
-        sql.append("SET")
-        vals = []
-        connect = ""
-        for key, val in data.items():
-            if connect:
-                sql.append(connect)
-            if isinstance(val, str) and val[:2] == "@@" and val[2:]:
-                sql.append(f"{key} = {val[2:]}")
-            else:
-                if excluded:
-                    sql.append(f"{key} = EXCLUDED.{key}")
-                else:
-                    sql.append(f"{key} = %s")
-                    vals.append(val)
-            connect = ","
-        if is_join:
-            sql.append("FROM")
-            sql.append(table)
-            sql.append("AS")
-            sql.append(alias)
-        if left_join:
-            for k, v in left_join.items():
-                sql.append("LEFT JOIN")
-                sql.append(k)
-                sql.append("ON")
-                sql.append(v)
-        if outer_join:
-            for k, v in outer_join.items():
-                sql.append("OUTER JOIN")
-                sql.append(k)
-                sql.append("ON")
-                sql.append(v)
-        if inner_join:
-            for k, v in inner_join.items():
-                sql.append("INNER JOIN")
-                sql.append(k)
-                sql.append("ON")
-                sql.append(v)
-        if not excluded:
-            make_where(pk, sql, vals, is_join)
-        return " ".join(sql), tuple(vals)
-
-    @classmethod
-    def get_type(cls, v):
-        if isinstance(v, str):
-            if v[:2] == "@@":
-                return v[2:] or cls.TYPES.TEXT
-        elif isinstance(v, str) or v is str:
-            return cls.TYPES.TEXT
-        elif isinstance(v, bool) or v is bool:
-            return cls.TYPES.BOOLEAN
-        elif isinstance(v, int) or v is int:
-            return cls.TYPES.BIGINT
-        elif isinstance(v, float) or v is float:
-            return f"{cls.TYPES.NUMERIC}(19, 6)"
-        elif isinstance(v, decimal.Decimal) or v is decimal.Decimal:
-            return f"{cls.TYPES.NUMERIC}(19, 6)"
-        elif isinstance(v, datetime.datetime) or v is datetime.datetime:
-            return cls.TYPES.DATETIME
-        elif isinstance(v, datetime.date) or v is datetime.date:
-            return cls.TYPES.DATE
-        elif isinstance(v, datetime.time) or v is datetime.time:
-            return cls.TYPES.TIME
-        elif isinstance(v, datetime.timedelta) or v is datetime.timedelta:
-            return cls.TYPES.INTERVAL
-        elif isinstance(v, bytes) or v is bytes:
-            return cls.TYPES.BINARY
-        return cls.TYPES.TEXT
-
-    @classmethod
-    def get_conv(cls, v):
-        if isinstance(v, str):
-            if v[:2] == "@@":
-                return v[2:] or cls.TYPES.TEXT
-        elif isinstance(v, str) or v is str:
-            return cls.TYPES.TEXT
-        elif isinstance(v, bool) or v is bool:
-            return cls.TYPES.BOOLEAN
-        elif isinstance(v, int) or v is int:
-            return cls.TYPES.BIGINT
-        elif isinstance(v, float) or v is float:
-            return cls.TYPES.NUMERIC
-        elif isinstance(v, decimal.Decimal) or v is decimal.Decimal:
-            return cls.TYPES.NUMERIC
-        elif isinstance(v, datetime.datetime) or v is datetime.datetime:
-            return cls.TYPES.DATETIME
-        elif isinstance(v, datetime.date) or v is datetime.date:
-            return cls.TYPES.DATE
-        elif isinstance(v, datetime.time) or v is datetime.time:
-            return cls.TYPES.TIME
-        elif isinstance(v, datetime.timedelta) or v is datetime.timedelta:
-            return cls.TYPES.INTERVAL
-        elif isinstance(v, bytes) or v is bytes:
-            return cls.TYPES.BINARY
-        return cls.TYPES.TEXT
-
-    @classmethod
-    def py_type(cls, v):
-        v = str(v).upper()
-        if v == cls.TYPES.INTEGER:
-            return int
-        elif v == cls.TYPES.SMALLINT:
-            return int
-        elif v == cls.TYPES.BIGINT:
-            return int
-        elif v == cls.TYPES.NUMERIC:
-            return decimal.Decimal
-        elif v == cls.TYPES.TEXT:
-            return str
-        elif v == cls.TYPES.BOOLEAN:
-            return bool
-        elif v == cls.TYPES.DATE:
-            return datetime.date
-        elif v == cls.TYPES.TIME:
-            return datetime.time
-        elif v == cls.TYPES.TIME_TZ:
-            return datetime.time
-        elif v == cls.TYPES.DATETIME:
-            return datetime.datetime
-        elif v == cls.TYPES.INTERVAL:
-            return datetime.timedelta
-        elif v == cls.TYPES.DATETIME_TZ:
-            return datetime.datetime
-        elif v == cls.TYPES.INTERVAL_TZ:
-            return datetime.timedelta
-        else:
-            raise Exception(f"unmapped type {v}")
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple()
 
     @classmethod
     def massage_data(cls, data):
@@ -985,46 +901,53 @@ class SQL(object):
             for key, val in columns.items():
                 key = re.sub("<>!=%", "", key.lower())
                 sql.append(
-                    f"ALTER TABLE {quote(table)} ADD {quote(key)} {cls.get_type(val)} {null};"
+                    f"ALTER TABLE {TableHelper.quote(table)} ADD {TableHelper.quote(key)} {TYPES.get_type(val)} {null};"
                 )
-        return "\n\t".join(sql), tuple()
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple()
 
     @classmethod
     def alter_drop(cls, table, columns):
-        sql = [f"ALTER TABLE {quote(table)} DROP COLUMN"]
+        sql = [f"ALTER TABLE {TableHelper.quote(table)} DROP COLUMN"]
         if isinstance(columns, dict):
             for key, val in columns.items():
                 key = re.sub("<>!=%", "", key.lower())
                 sql.append(f"{key},")
         if sql[-1][-1] == ",":
             sql[-1] = sql[-1][:-1]
-        return "\n\t".join(sql), tuple()
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple()
 
     @classmethod
     def alter_column_by_type(cls, table, column, value, nullable=True):
-        sql = [f"ALTER TABLE {quote(table)} ALTER COLUMN"]
-        sql.append(f"{quote(column)} TYPE {cls.get_type(value)}")
-        sql.append(f"USING {quote(column)}::{cls.get_conv(value)}")
+        sql = [f"ALTER TABLE {TableHelper.quote(table)} ALTER COLUMN"]
+        sql.append(f"{TableHelper.quote(column)} TYPE {TYPES.get_type(value)}")
+        sql.append(f"USING {TableHelper.quote(column)}::{TYPES.get_conv(value)}")
         if not nullable:
             sql.append("NOT NULL")
-        return "\n\t".join(sql), tuple()
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple()
 
     @classmethod
     def alter_column_by_sql(cls, table, column, value):
-        sql = [f"ALTER TABLE {quote(table)} ALTER COLUMN"]
-        sql.append(f"{quote(column)} {value}")
-        return " ".join(sql), tuple()
+        sql = [f"ALTER TABLE {TableHelper.quote(table)} ALTER COLUMN"]
+        sql.append(f"{TableHelper.quote(column)} {value}")
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple()
 
     @classmethod
     def rename_column(cls, table, orig, new):
         return (
-            f"ALTER TABLE {quote(table)} RENAME COLUMN {quote(orig)} TO {quote(new)};",
+            f"ALTER TABLE {TableHelper.quote(table)} RENAME COLUMN {TableHelper.quote(orig)} TO {TableHelper.quote(new)};",
             tuple(),
         )
 
     @classmethod
     def rename_table(cls, table, new):
-        return f"ALTER TABLE {quote(table)} RENAME TO {quote(new)};", tuple()
+        return (
+            f"ALTER TABLE {TableHelper.quote(table)} RENAME TO {TableHelper.quote(new)};",
+            tuple(),
+        )
 
     @classmethod
     def create_savepoint(cls, sp):
@@ -1039,26 +962,19 @@ class SQL(object):
         return f'ROLLBACK TO SAVEPOINT "{sp}"', tuple()
 
     @classmethod
-    def duplicate_rows(cls, table, columns, where={}):
-        return cls.select(
-            columns,
-            table,
-            where,
-            orderby=columns,
-            groupby=columns,
-            having="count(*) > 2",
-        )
-
-    @classmethod
-    def delete(cls, table, where):
+    def delete(cls, tx, table, where):
         sql = [f"DELETE FROM {table}"]
         vals = []
-        make_where(where, sql, vals)
-        return " ".join(sql), tuple(vals)
+        if where:
+            s, v = TableHelper(tx, table).make_where(where)
+            sql.append(s)
+            vals.extend(v)
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple(vals)
 
     @classmethod
     def truncate(cls, table):
-        return f"truncate table {quote(table)}", tuple()
+        return f"truncate table {TableHelper.quote(table)}", tuple()
 
     @classmethod
     def create_view(cls, name, query, temp=False, silent=True):
@@ -1071,7 +987,8 @@ class SQL(object):
         sql.append(name)
         sql.append("AS")
         sql.append(query)
-        return " ".join(sql), tuple()
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple()
 
     @classmethod
     def drop_view(cls, name, silent=True):
@@ -1079,7 +996,8 @@ class SQL(object):
         if silent:
             sql.append("IF EXISTS")
         sql.append(name)
-        return " ".join(sql), tuple()
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple()
 
     @classmethod
     def alter_trigger(cls, table, state="ENABLE", name="USER"):
@@ -1093,7 +1011,7 @@ class SQL(object):
         )
 
     @classmethod
-    def missing(cls, table, list, column="SYS_ID", where=None):
+    def missing(cls, tx, table, list, column="SYS_ID", where=None):
         sql = [
             f"SELECT * FROM",
             f"UNNEST('{{{','.join([str(x) for x in list])}}}'::int[]) id",
@@ -1101,22 +1019,23 @@ class SQL(object):
             f"SELECT {column} FROM {table}",
         ]
         vals = []
-        make_where(where, sql, vals)
-        return " ".join(sql), tuple(vals)
-
-    class TYPES(object):
-        TEXT = "TEXT"
-        INTEGER = "INTEGER"
-        NUMERIC = "NUMERIC"
-        DATETIME_TZ = "TIMESTAMP WITH TIME ZONE"
-        TIMESTAMP_TZ = "TIMESTAMP WITH TIME ZONE"
-        DATETIME = "TIMESTAMP WITHOUT TIME ZONE"
-        TIMESTAMP = "TIMESTAMP WITHOUT TIME ZONE"
-        DATE = "DATE"
-        TIME_TZ = "TIME WITH TIME ZONE"
-        TIME = "TIME WITHOUT TIME ZONE"
-        BIGINT = "BIGINT"
-        SMALLINT = "SMALLINT"
-        BOOLEAN = "BOOLEAN"
-        BINARY = "BYTEA"
-        INTERVAL = "INTERVAL"
+        if where:
+            s, v = TableHelper(tx, table).make_where(where)
+            sql.append(s)
+            vals.extend(v)
+        sql = sqlparse.format(" ".join(sql), reindent=True, keyword_case="upper")
+        return sql, tuple(vals)
+
+    @classmethod
+    def indexes(cls, table):
+        """
+        Returns SQL for retrieving all indexes on a given table with detailed attributes.
+        """
+        return (
+            """
+            SELECT indexname, tablename, schemaname, indexdef
+            FROM pg_indexes
+            WHERE tablename = %s
+            """,
+            (table,),
+        )