pydantic-table 0.3.0__py3-none-any.whl

pydantic_table/__init__.py

@@ -0,0 +1,4 @@
+from pydantic_table.__version__ import __version__
+
+from pydantic_table.table_model.field import ColumnField as ColumnField
+from pydantic_table.table_model.model import TableModel as TableModel

pydantic_table/__version__.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+
+__version__ = version("pydantic-table")

pydantic_table/alembic/archive.py

@@ -0,0 +1,162 @@
+import json
+import os
+from pathlib import Path
+from typing import Any, Type
+
+import inspect
+
+from alembic import op
+
+from pydantic_table.alembic.exceptions import ArchiveException
+from pydantic_table.logger import logg
+from pydantic_table.table_model.field import ColumnFieldInfo
+from pydantic_table.table_model.model import TableModel
+
+import sqlalchemy as sa
+import pydantic_table.sqlalchemy as sap
+
+SAVE_DATA = False
+
+
+class Archive:
+    def __init__(self) -> None:
+        self._revision_filepath = self._get_revision_filepath()
+
+        versions = self._revision_filepath.parent
+        self._dir = versions / ".archive"
+
+        if not self._dir.exists():
+            os.makedirs(self._dir)
+
+        revision_filename = Path(self._revision_filepath).stem
+        self._archive_file = self._dir / f"{revision_filename}.json"
+        logg.debug(f"archive file: {self._archive_file}")
+
+    @property
+    def file_exists(self) -> bool:
+        return self._archive_file.exists()
+
+    def archive_table_model(self, sa_table: sa.Table):
+        """
+        Archive table schema.
+        """
+        # TODO: check if content exists, not if file exists
+        if self.file_exists:
+            logg.debug("-> already exists")
+            return
+
+        table_dict = {}
+        for name, column in sa_table.c.items():
+            table_dict[name] = self._build_column_info_dict(column)
+
+        archive_dict = {sa_table.name: table_dict}
+        self._save_archive_dict(archive_dict)
+
+    def read_column_fields(self, table_name: str) -> dict[str, ColumnFieldInfo]:
+        """
+        Read table model from archive.
+
+        Return map of column field infos for each column name (i.e. TableModel.column_fields() content)
+        """
+        column_fields = self._check_and_load_file_content(content=table_name)
+        ret = {
+            column_name: self._build_column_info(column_info)
+            for column_name, column_info in column_fields.items()
+        }
+        return ret
+
+    # TODO: multiple columns
+    def archive_column_info(self, table: Type[TableModel], column_name: str):
+        """
+        Archive column information.
+
+        Extract column field information from table (not table model, which does not have it anymore).
+        Extract path to the migration file (assumes drop_column is called from a migration)
+        Create archive directory if does not exist.
+        Serialize field information to be saved in archive JSON.
+        Get dict data from sa.Table to archive.
+        """
+
+        # TODO: check if content exists, not if file exists
+        if self.file_exists:
+            logg.debug("-> already exists")
+            return
+
+        engine = op.get_bind()
+
+        tb = sap.Table(table, autoload_with=engine)
+        sa_column = tb.c[column_name]
+        archive_dict: dict[str, dict[str, Any]] = {
+            column_name: {"info": self._build_column_info_dict(sa_column)}
+        }
+
+        if SAVE_DATA:
+            result = engine.execute(tb.select())
+            data_dict = [dict(row) for row in result.mappings()]
+            archive_dict[column_name]["data"] = data_dict
+
+        self._save_archive_dict(archive_dict)
+
+    def read_column_info(self, column: str) -> ColumnFieldInfo:
+        """
+        Get column schema from archive
+        """
+        column_dict = self._check_and_load_file_content(content=column)
+
+        if not "info" in column_dict:
+            raise ArchiveException(
+                f"Field 'info' not found for column {column} in archive file {self._archive_file}!"
+            )
+
+        ret = self._build_column_info(column_dict["info"])
+        return ret
+
+    def _build_column_info_dict(self, column: sa.Column) -> dict[str, Any]:
+        """
+        Build column field info dict based on sqlalchemy column.
+        """
+        column_info = sap.ColumnFieldInfo(column)
+        ret = column_info.as_dict()
+        ret["annotation"] = ret["annotation"].__name__
+        return ret
+
+    def _build_column_info(self, column_info: dict) -> ColumnFieldInfo:
+        info_dict = column_info.copy()
+
+        info_dict["annotation"] = eval(info_dict["annotation"])
+        ret = ColumnFieldInfo(**info_dict)
+        return ret
+
+    def _save_archive_dict(self, dct: dict[str, Any]):
+        logg.debug(f"archive dict: {dct}")
+
+        with open(self._archive_file, "w") as f:
+            json.dump(dct, f, indent=2)
+
+    def _check_and_load_file_content(self, content: str) -> dict:
+        if not self._archive_file.exists():
+            raise ArchiveException(f"Archive file {self._archive_file} not found!")
+
+        with open(self._archive_file) as f:
+            file_dict = json.load(f)
+
+        if not content in file_dict:
+            raise ArchiveException(
+                f"{content} does not exist in archive file {self._archive_file}!"
+            )
+
+        ret = file_dict[content]
+        return ret
+
+    def _get_revision_filepath(self) -> Path:
+        frame = inspect.currentframe()
+        assert frame is not None, "got null frame"
+        archive_caller = frame.f_back
+        assert archive_caller is not None, "no archive caller frame"
+        opp_caller = archive_caller.f_back
+        assert opp_caller is not None, "no opp caller frame"
+        revision_caller = opp_caller.f_back
+        assert revision_caller is not None, "no revision caller frame"
+
+        ret = Path(revision_caller.f_code.co_filename)
+        return ret

pydantic_table/alembic/exceptions.py

@@ -0,0 +1,5 @@
+class PydanticTableAlembicException(Exception):
+    pass
+
+class ArchiveException(PydanticTableAlembicException):
+    pass

pydantic_table/alembic/op.py

@@ -0,0 +1,241 @@
+"""
+op adaptors
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from typing import Any, Type
+
+from pydantic_table.alembic.archive import Archive
+from pydantic_table.alembic.exceptions import (
+    ArchiveException,
+    PydanticTableAlembicException,
+)
+from pydantic_table.logger import logg
+import pydantic_table.sqlalchemy as sap
+
+from pydantic_table.table_model.model import TableModel
+from pydantic_table.utils import dict_as_str
+
+
+def create_table(
+    table: Type[TableModel],
+    foreign_keys: dict[str, str] = {},
+):
+    """
+    Invoke alembic create table based on provided TableModel schema.
+
+    primary_keys (list[str]): list of primary key column names
+    foreign_keys (dict[str, str]): dictionary mapping {column name: foreign key column information}
+        Format of foreign key is foreign_table.column
+
+    Column names must correspond to TableModel field names.
+    """
+    archive = Archive()
+    if archive.file_exists:
+        try:
+            # TODO: table rename
+            column_fields = archive.read_column_fields(table.table_name())
+        except ArchiveException:
+            raise PydanticTableAlembicException(
+                f"Table {table.table_info()} schema has changed relative to this revision but archive file not found! Cannot create table."
+            )
+    else:
+        column_fields = table.column_fields()
+        
+    sa_columns = [
+        sap.Column(name, column_info, foreign_key=foreign_keys.get(name, None))
+        for name, column_info in column_fields.items()
+    ]
+
+    op.create_table(table.table_name(), *sa_columns)
+
+
+def drop_table(table: Type[TableModel]):
+    """
+    Invoke alembic drop table based on provided TableModel schema.
+
+    If detect extra columns or missing columns i.e. any schema change,
+        archive table schema.
+    """
+    sa_table = sap.Table(table, autoload_with=op.get_bind())
+    # TODO: mutual set difference
+    # TODO: detect any schema change in column info (changed default, changed nullability or primary)
+    model_has_new_columns = any(
+        col_name not in table.column_fields() for col_name in sa_table.c
+    )
+    model_is_missing_columns = any(
+        col_name not in sa_table.c for col_name in table.column_fields()
+    )
+    if model_has_new_columns or model_is_missing_columns:
+        Archive().archive_table_model(sa_table)
+
+    op.drop_table(table.table_name())
+
+
+def update_where(table: Type[TableModel], values: dict[str, Any], **kwargs):
+    """
+    Update table with given values {column: value}.
+
+    kwargs in format column=value for the where condition
+    """
+    tb = sap.Table(table, autoload_with=op.get_bind())
+    condition = _get_condition(tb, **kwargs)
+    logg.debug(f"Values: {dict_as_str(values)}")
+    op.execute(tb.update().where(condition).values(values))
+
+
+def add_column(
+    table: Type[TableModel],
+    name: str,
+    data: list[TableModel] | TableModel = [],
+    foreign_key: str | None = None,
+):
+    """
+    Add column to table.
+
+    Given name must be present in table column fields or in archive.
+    Fill column with given data; otherwise initialize to column default (must be present).
+
+    In case column is not nullable, no default, and data is given:
+        - create column first as nullable to prevent crash
+        - add given column data (use other columns for condition)
+        - set column back to nullable
+    """
+    archive = Archive()
+    if archive.file_exists:
+        try:
+            column_info = archive.read_column_info(name)
+        except ArchiveException:
+            raise PydanticTableAlembicException(
+                f"Column {name} not present in table {table.table_info()} or in archive! Cannot add."
+            )
+    else:
+        column_info = table.column_fields()[name]
+
+    sa_column = sap.Column(name, column_info, foreign_key=foreign_key)
+
+    logg.debug(f"Adding column - column info: {column_info}")
+    logg.debug(
+        f"default={column_info.default} default factory={column_info.default_factory} required={column_info.is_required()}"
+    )
+    # is required = neither default nor default factory are defined
+    if not column_info.nullable and column_info.is_required():
+        data_list = data if isinstance(data, list) else [data]
+        if len(data_list) == 0:
+            raise PydanticTableAlembicException(
+                f"Column {name} in {table.table_info()} is not nullable and no default was given. You provided no data."
+                f"Either provide data to add, or a default; or make column nullable"
+            )
+
+        sa_column.nullable = True
+        op.add_column(table.table_name(), sa_column)
+        for row in data_list:
+            logg.debug(f"Adding data row - {row}")
+            logg.debug(f"Column dump - {row.column_dump()}")
+            logg.debug(f"Data dump - {row.data_dump()}")
+            update_where(
+                table,
+                values={name: row.get(name)},
+                **row.column_dump(exclude={name: True}),
+            )
+        # op.alter_column(table.table_name(), name, nullable=False) - error
+        with op.batch_alter_table(table.table_name()) as batch_op:
+            batch_op.alter_column(name, nullable=False)
+        return
+
+    op.add_column(
+        table.table_name(), sap.Column(name, column_info, foreign_key=foreign_key)
+    )
+
+
+def drop_column(table: Type[TableModel], name: str):
+    """
+    Drop column.
+
+    Drop column from the table.
+    Archive column if not present in TableModel anymore
+        (schema update removes column)
+    """
+    if name not in table.column_fields():
+        Archive().archive_column_info(table, name)
+
+    op.drop_column(table.table_name(), name)
+
+
+def insert(rows: TableModel | list[TableModel]):
+    """
+    Insert given row(s) to the table corresponding to its schema.
+
+    Read table based on table name.
+    Invoke alembic execute() of table insert() with model dump.
+
+    Account for backwards compatibility with added columns or dropped columns.
+
+    Check existing columns in input that were not given (missing columns):
+        - if not present in table, ignore (past/future schema change dropped/added this column)
+        - if present in table, raise error (must be given but was not)
+
+    Ignore columns that are not present in table.
+
+    Check hidden extra columns in input (extra columns):
+        - if present in table, include (past/future schema change added/dropped this column)
+        - if not present in table, raise error (non-existing columns were given)
+    """
+    row_list = rows if isinstance(rows, list) else [rows]
+    for row in row_list:
+        table = sap.Table(row.table, autoload_with=op.get_bind())
+
+        for col_name in row.missing_columns:
+            if col_name in table.c:
+                raise PydanticTableAlembicException(
+                    f"Row given for table {row.table_name()} is missing column {col_name}!"
+                )
+
+        data = row.column_dump()
+        logg.debug(f"Inserting data - column dump: {data}")
+        data = {col: val for col, val in data.items() if col in table.c}
+        logg.debug(f"Inserting data - only columns in table: {data}")
+
+        for col_name, col_value in row.extra_data.items():
+            if col_name not in table.c:
+                raise PydanticTableAlembicException(
+                    f"Row given for table {row.table_info()} has extra column {col_name}!"
+                )
+            data[col_name] = col_value
+
+        logg.debug(f"Inserting data + extra columns: {data}")
+        op.execute(table.insert().values(data))
+
+
+def delete_where(table: Type[TableModel], **kwargs):
+    """
+    Delete rows from table where columns have given values.
+
+    kwargs in format column=value for the where condition.
+    """
+    tb = sap.Table(table, autoload_with=op.get_bind())
+    condition = _get_condition(tb, **kwargs)
+    op.execute(tb.delete().where(condition))
+
+
+def deep_delete(rows: TableModel | list[TableModel]):
+    """
+    Delete given row(s) from the table corresponding to its schema.
+
+    Read table based on table name.
+    Invoke alembic execute() with table delete() matching all fields in where().
+
+    Note that this operation takes a lot of time, but is the most secure as it deletes exactly the row.
+    """
+    row_list = rows if isinstance(rows, list) else [rows]
+    table = row_list[0].table
+
+    for row in row_list:
+        delete_where(table, **row.data_dump())
+
+
+def _get_condition(tb: sa.Table, **kwargs) -> sa.ColumnElement[bool]:
+    logg.debug(f"Condition: {dict_as_str(kwargs)}")
+    condition = sa.and_(*[tb.c[column] == value for column, value in kwargs.items()])
+    return condition

pydantic_table/logger.py

@@ -0,0 +1,136 @@
+import enum
+import inspect
+import logging
+from pathlib import Path
+from typing import Any
+from dotenv import dotenv_values
+
+
+class AnsiStyle(str, enum.Enum):
+    normal = "0"
+    bold = "1"
+    start = "\033["
+    end = "\033[0m"
+
+    def __str__(self) -> str:
+        return self.value
+
+
+class AnsiColor(str, enum.Enum):
+    green = "32"
+    grey = "90"
+    red = "31"
+    yellow = "33"
+    white = "37"
+
+    def apply(self, message: Any, bold: bool = False) -> str:
+        """
+        To be used with color based
+        """
+        style = AnsiStyle.bold if bold else AnsiStyle.normal
+        ret = f"{AnsiStyle.start}{style};{self.value}m{message}{AnsiStyle.end}"
+        return ret
+
+    def bold(self, message: Any) -> str:
+        """
+        Shortcut for bold colored text
+        """
+        ret = self.apply(message, bold=True)
+        return ret
+
+    def __str__(self) -> str:
+        return self.value
+
+
+class LevelFormatter(logging.Formatter):
+    def __init__(self, formats, default_fmt=None, datefmt=None):
+        super().__init__(datefmt=datefmt)
+        self.formats = {
+            level: logging.Formatter(fmt, datefmt=datefmt)
+            for level, fmt in formats.items()
+        }
+        self.default_formatter = logging.Formatter(
+            default_fmt or "%(message)s", datefmt=datefmt
+        )
+
+    def format(self, record):
+        formatter = self.formats.get(record.levelno, self.default_formatter)
+        return formatter.format(record)
+
+
+class Logger:
+    def __init__(self, log_level=logging.INFO):
+        self._logger = logging.getLogger(__name__)
+
+        full_format = "%(asctime)s [%(levelname)s] %(classname)s.%(funcName)s:%(lineno)d - %(message)s"
+        short_format = "[%(levelname)s] %(classname)s - %(message)s"
+        no_format = ""
+        level_format = full_format if log_level == logging.DEBUG else no_format
+        formatter = LevelFormatter(
+            {
+                logging.DEBUG: full_format,
+                logging.ERROR: level_format,
+                logging.INFO: level_format,
+                logging.WARNING: level_format,
+            }
+        )
+
+        handler = logging.StreamHandler()
+        handler.setFormatter(formatter)
+        handler.setLevel(log_level)
+        self._logger.addHandler(handler)
+
+        self._logger.setLevel(log_level)
+        self._logger.propagate = False
+
+    def info(self, message: Any, header: bool = False):
+        color = AnsiColor.green if header else AnsiColor.white
+        return self._log(logging.INFO, color.apply(message, header))
+
+    def error(self, message: Any):
+        return self._log(logging.ERROR, AnsiColor.red.apply(message))
+
+    def warning(self, message: Any, important: bool = False):
+        if important:
+            message = f"! WARNING ! {message}"
+        return self._log(logging.WARNING, AnsiColor.yellow.apply(message, important))
+
+    def debug(self, message: Any):
+        return self._log(logging.DEBUG, AnsiColor.grey.apply(message))
+
+    def _log(self, level, message: Any):
+        """
+        Common log interface for info/error/warning/debug.
+
+        Auto-detect class name of caller.
+        Account for stack level to display correct funcName and lineno.
+            Skip 2 stack levels including this method,
+            the info/error/warning/debug method calling it,
+        """
+        self._logger.log(
+            level,
+            message,
+            extra={"classname": self._get_caller_class_name()},
+            stacklevel=3,
+        )
+
+    def _get_caller_class_name(self):
+        """
+        Determine class name of where logger is called from.
+
+        Obtain frame index 3 corresponding to actual caller
+            (0 = this method, 1 = _log internal method, 2 = info/warning/error/debug logger call).
+        Return class or module name of that frame.
+        """
+        frame = inspect.stack()[3].frame
+        if "self" in frame.f_locals:
+            return type(frame.f_locals["self"]).__name__
+        elif "cls" in frame.f_locals:
+            return frame.f_locals["cls"].__name__
+        return frame.f_globals.get("__name__", "-")
+
+
+# NOTE: dotenv_values() in some cases yielded empty .env for unexplained reason
+env_config = dotenv_values(Path.cwd() / ".env")
+is_debug = env_config.get("DEBUG_PYDANTIC_TABLE", "").lower() in ("true", "1")
+logg = Logger(log_level=logging.DEBUG if is_debug else logging.INFO)

pydantic_table/sqlalchemy/__init__.py

@@ -0,0 +1,3 @@
+from pydantic_table.sqlalchemy.base import BaseMeta as BaseMeta
+from pydantic_table.sqlalchemy.column import Column as Column, ColumnFieldInfo as ColumnFieldInfo
+from pydantic_table.sqlalchemy.table import Table as Table

pydantic_table/sqlalchemy/base.py

@@ -0,0 +1,57 @@
+import enum
+from typing import Any, Type
+
+from sqlalchemy import Float, Integer, String
+from sqlalchemy.orm import DeclarativeBase, mapped_column
+from sqlalchemy.orm.decl_api import DeclarativeAttributeIntercept
+
+from pydantic_table.table_model.model import TableModel
+
+
+class BaseFieldType(enum.Enum):
+    int = Integer()
+    float = Float(32)
+    str = String(255)
+
+    @classmethod
+    def from_type(cls, type: type):
+        return cls[type.__name__].value
+
+
+class BaseMeta(DeclarativeAttributeIntercept):
+    """
+    Adaptor Metaclass for creating DeclarativeBase based on pydantic-table TableModel.
+
+    Translates TableModel field:
+        - table_name_ --> __tablename__
+        - annotation -> sqlalchemy type (Integer, Float, String)
+        - default=None -> nullable=True
+    """
+
+    def __new__(
+        cls,
+        name: str,
+        bases: tuple[type, ...],
+        namespace: dict[str, Any],
+        /,
+        model: Type[TableModel],
+        **kwds: Any,
+    ):
+        namespace["__tablename__"] = model.table_name()
+
+        for column_name, column_info in model.column_fields().items():
+            if column_info.annotation is None:
+                # TODO: model validator (this should basically be assert level)
+                raise ValueError(
+                    f"pydantic model fields must be annotated for pydantic-table sqlalchemy adaptor!\n{column_info}"
+                )
+
+            namespace[column_name] = mapped_column(
+                BaseFieldType.from_type(column_info.annotation),
+                nullable=column_info.nullable,
+                primary_key=column_info.primary_key,
+            )
+
+        x = super().__new__(cls, name, bases, namespace, **kwds)
+
+        return x