database-wrapper 0.1.44__py3-none-any.whl → 0.1.72__py3-none-any.whl

database_wrapper/__init__.py

@@ -6,12 +6,15 @@ database_wrapper package - Base for database wrappers
 
 import logging
 
+from .abc import ConnectionABC, CursorABC, CursorAsyncABC, ConnectionAsyncABC
 from . import utils
 from .db_backend import DatabaseBackend
-from .db_data_model import DBDataModel, DBDefaultsDataModel
+from .db_data_model import MetadataDict, DBDataModel, DBDefaultsDataModel
 from .common import OrderByItem, DataModelType, NoParam
 from .db_wrapper import DBWrapper
 from .db_wrapper_async import DBWrapperAsync
+from .serialization import SerializeType
+from .utils.dataclass_addons import ignore_unknown_kwargs
 
 # Set the logger to a quiet default, can be enabled if needed
 logger = logging.getLogger("database_wrapper")
@@ -30,8 +33,16 @@ __all__ = [
     "DBWrapper",
     "DBWrapperAsync",
     # Helpers
+    "MetadataDict",
     "DataModelType",
     "OrderByItem",
     "NoParam",
     "utils",
+    "SerializeType",
+    "ignore_unknown_kwargs",
+    # Abstract classes
+    "ConnectionABC",
+    "CursorABC",
+    "CursorAsyncABC",
+    "ConnectionAsyncABC",
 ]

database_wrapper/abc.py

@@ -0,0 +1,68 @@
+from typing import Dict, Any, Literal
+from typing import Any, Protocol, Sequence
+
+from psycopg.sql import Composable
+
+Row = Dict[str, Any]
+
+
+class CursorABC(Protocol):
+    def execute(
+        self,
+        operation: str | Composable,
+        parameters: Sequence[Any] | None = None,
+    ) -> None: ...
+    def executemany(
+        self,
+        operation: str | Composable,
+        seq_of_parameters: Sequence[Sequence[Any]],
+    ) -> None: ...
+    def fetchone(self) -> Row | None: ...
+    def fetchmany(self, size: int | None = None) -> list[Row]: ...
+    def fetchall(self) -> list[Row]: ...
+    def close(self) -> None: ...
+    def setinputsizes(self, sizes: Sequence[Any]) -> None: ...
+    def setoutputsize(self, size: Any, column: Any = None) -> None: ...
+
+    @property
+    def rowcount(self) -> int | Literal[-1]: ...
+    @property
+    def lastrowid(self) -> int | None: ...
+
+
+class ConnectionABC(Protocol):
+    def cursor(self) -> CursorABC: ...
+    def commit(self) -> None: ...
+    def rollback(self) -> None: ...
+    def close(self) -> None: ...
+
+
+class CursorAsyncABC(Protocol):
+    async def execute(
+        self,
+        operation: str | Composable,
+        parameters: Sequence[Any] | None = None,
+    ) -> None: ...
+    async def executemany(
+        self,
+        operation: str | Composable,
+        seq_of_parameters: Sequence[Sequence[Any]],
+    ) -> None: ...
+    async def fetchone(self) -> Row | None: ...
+    async def fetchmany(self, size: int | None = None) -> list[Row]: ...
+    async def fetchall(self) -> list[Row]: ...
+    async def close(self) -> None: ...
+    def setinputsizes(self, sizes: Sequence[Any]) -> None: ...
+    def setoutputsize(self, size: Any, column: Any = None) -> None: ...
+
+    @property
+    def rowcount(self) -> int | Literal[-1]: ...
+    @property
+    def lastrowid(self) -> int | None: ...
+
+
+class ConnectionAsyncABC(Protocol):
+    async def cursor(self) -> CursorAsyncABC: ...
+    async def commit(self) -> None: ...
+    async def rollback(self) -> None: ...
+    async def close(self) -> None: ...

database_wrapper/config.py

@@ -3,7 +3,7 @@ from typing import Any
 CONFIG: dict[str, Any] = {
     # These are supposed to be set automatically by a git pre-compile script
     # They are one git commit hash behind, if used automatically
-    "git_commit_hash": "ec2eca45cd9798f0c045418602fbe07ed7734848",
-    "git_commit_date": "15.11.2024 18:06",
-    "app_version": "0.1.44",
+    "git_commit_hash": "943e0234156c4a164061e5af9fb9dbd8c3041f6f",
+    "git_commit_date": "26.11.2024 22:09",
+    "app_version": "0.1.72",
 }

database_wrapper/db_backend.py

@@ -5,63 +5,114 @@ from typing import Any
 from threading import Event
 from contextvars import ContextVar
 
-from .utils.timer import Timer
-
 
 class DatabaseBackend:
+    config: Any
+    """ Database configuration """
+
+    connectionTimeout: int
+    """ Connection timeout """
+
+    name: str
+    """ Instance name """
+
+    # TODO: This should be made to increase exponentially
+    slowDownTimeout: int
+    """ How long to wait before trying to reconnect """
+
+    pool: Any
+    """ Connection pool """
+
+    poolAsync: Any
+    """ Async connection pool """
+
     connection: Any
+    """ Connection to database """
+
     cursor: Any
+    """ Cursor to database """
+
     contextConnection: ContextVar[Any | None]
-    contextAsyncConnection: ContextVar[Any | None]
+    """ Connection used in context manager """
 
-    config: Any
+    contextConnectionAsync: ContextVar[Any | None]
+    """ Connection used in async context manager """
 
-    connectionTimeout: int
-    slowDownTimeout: int = 5
+    loggerName: str
+    """ Logger name """
 
-    name: str
     logger: logging.Logger
-    timer: ContextVar[Timer | None]
+    """ Logger """
 
     shutdownRequested: Event
+    """
+    Event to signal shutdown
+    Used to stop database pool from creating new connections
+    """
+
+    ########################
+    ### Class Life Cycle ###
+    ########################
 
     def __init__(
         self,
         dbConfig: Any,
         connectionTimeout: int = 5,
         instanceName: str = "database_backend",
+        slowDownTimeout: int = 5,
     ) -> None:
         """
         Main concept here is that in init we do not connect to database,
         so that class instances can be safely made regardless of connection statuss.
 
-        Remember to call open() before using this class.
+        Remember to call open() or openPool() before using this class.
         Close will be called automatically when class is destroyed.
-        But sometimes in async environment you should call close() proactively.
+
+        Contexts are not implemented here, but in child classes should be used
+        by using connection pooling.
+
+        Async classes should be called manually and should override __del__ method,
+        if not upon destroying the class, an error will be raised that method was not awaited.
         """
 
         self.config = dbConfig
         self.connectionTimeout = connectionTimeout
         self.name = instanceName
+        self.slowDownTimeout = slowDownTimeout
+
+        self.loggerName = f"{__name__}.{self.__class__.__name__}.{self.name}"
+        self.logger = logging.getLogger(self.loggerName)
 
-        loggerName = f"{__name__}.{self.__class__.__name__}.{self.name}"
-        self.logger = logging.getLogger(loggerName)
-        self.timer = ContextVar(f"db_timer", default=None)
+        self.pool = None
+        self.poolAsync = None
 
         self.connection = None
         self.cursor = None
         self.shutdownRequested = Event()
         self.contextConnection = ContextVar(f"db_connection_{self.name}", default=None)
-        self.contextAsyncConnection = ContextVar(
+        self.contextConnectionAsync = ContextVar(
             f"db_connection_{self.name}_async", default=None
         )
 
     def __del__(self) -> None:
         """What to do when class is destroyed"""
         self.logger.debug("Dealloc")
+
+        # Clean up connections
         self.close()
+        self.closePool()
+
+        # Clean just in case
+        del self.connection
+        del self.cursor
+
+        del self.pool
+        del self.poolAsync
+
+    ###############
+    ### Context ###
+    ###############
 
-    # Context
     def __enter__(self) -> tuple[Any, Any]:
         """Context manager"""
         raise Exception("Not implemented")
@@ -78,16 +129,23 @@ class DatabaseBackend:
         """Context manager"""
         raise Exception("Not implemented")
 
-    # Connection
-    def open(self) -> None:
-        """Connect to database"""
-        raise Exception("Not implemented")
+    ##################
+    ### Connection ###
+    ##################
+
+    def openPool(self) -> Any:
+        """Open connection pool"""
+        ...
 
-    async def openAsync(self) -> None:
+    def closePool(self) -> Any:
+        """Close connection pool"""
+        ...
+
+    def open(self) -> Any:
         """Connect to database"""
-        raise Exception("Not implemented")
+        ...
 
-    def close(self) -> None:
+    def close(self) -> Any:
         """Close connections"""
         if self.cursor:
             self.logger.debug("Closing cursor")
@@ -99,7 +157,34 @@ class DatabaseBackend:
             self.connection.close()
             self.connection = None
 
-    def fixSocketTimeouts(self, fd: Any):
+    def newConnection(self) -> Any:
+        """
+        Create new connection
+
+        Used for async context manager and async connection creation
+
+        Returns:
+            tuple[Any, Any] | None: Connection and cursor
+        """
+        raise Exception("Not implemented")
+
+    def returnConnection(self, connection: Any) -> Any:
+        """
+        Return connection to pool
+
+        Used for async context manager and async connections return.
+        For example to return connection to a pool.
+
+        Args:
+            connection (Any): Connection to return to pool
+        """
+        raise Exception("Not implemented")
+
+    ###############
+    ### Helpers ###
+    ###############
+
+    def fixSocketTimeouts(self, fd: Any) -> None:
         # Lets do some socket magic
         s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
         # Enable sending of keep-alive messages
@@ -118,32 +203,36 @@ class DatabaseBackend:
             socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, self.connectionTimeout * 1000
         )
 
-    async def newConnection(
-        self,
-    ) -> tuple[Any, Any] | None:
-        """
-        Create new connection
+    ####################
+    ### Transactions ###
+    ####################
 
-        Used for async context manager and async connection creation
+    def beginTransaction(self) -> Any:
+        """Start transaction"""
+        raise Exception("Not implemented")
 
-        Returns:
-            tuple[Any, Any] | None: Connection and cursor
-        """
+    def commitTransaction(self) -> Any:
+        """Commit transaction"""
         raise Exception("Not implemented")
 
-    async def returnConnection(self, connection: Any) -> None:
-        """
-        Return connection to pool
+    def rollbackTransaction(self) -> Any:
+        """Rollback transaction"""
+        raise Exception("Not implemented")
 
-        Used for async context manager and async connections return.
-        For example to return connection to a pool.
+    # @contextmanager
+    def transaction(self, dbConn: Any = None) -> Any:
+        """
+        Transaction context manager
 
-        Args:
-            connection (Any): Connection to return to pool
+        ! When overriding this method, remember to use context manager.
+        ! Its not defined here, so that it can be used in both sync and async methods.
         """
         raise Exception("Not implemented")
 
-    # Data
+    ############
+    ### Data ###
+    ############
+
     def lastInsertId(self) -> int:
         """Get last inserted row id generated by auto increment"""
         raise Exception("Not implemented")
@@ -152,10 +241,10 @@ class DatabaseBackend:
         """Get affected rows count"""
         raise Exception("Not implemented")
 
-    def commit(self) -> None:
+    def commit(self) -> Any:
         """Commit DB queries"""
         raise Exception("Not implemented")
 
-    def rollback(self) -> None:
+    def rollback(self) -> Any:
         """Rollback DB queries"""
         raise Exception("Not implemented")

database_wrapper/db_data_model.py

@@ -1,12 +1,30 @@
+from enum import Enum
 import re
 import json
 import datetime
 import dataclasses
 
-from enum import Enum
 from dataclasses import dataclass, field, asdict
-from decimal import Decimal
-from typing import Any
+from typing import Any, Callable, Literal, NotRequired, Type, TypeVar, TypedDict, cast
+
+from .serialization import (
+    SerializeType,
+    deserializeValue,
+    jsonEncoder,
+    serializeValue,
+)
+
+EnumType = TypeVar("EnumType", bound=Enum)
+
+
+class MetadataDict(TypedDict):
+    db_field: tuple[str, str]
+    store: bool
+    update: bool
+    exclude: NotRequired[bool]
+    serialize: NotRequired[Callable[[Any], Any] | SerializeType | None]
+    deserialize: NotRequired[Callable[[Any], Any] | None]
+    enum_class: NotRequired[Type[Enum] | None]
 
 
 @dataclass
@@ -37,12 +55,12 @@ class DBDataModel:
     - validate(): Validates the instance.
 
     To enable storing and updating fields that by default are not stored or updated, use the following methods:
-    - setStore(field_name: str, enable: bool = True): Enable/Disable storing a field.
-    - setUpdate(field_name: str, enable: bool = True): Enable/Disable updating a field.
+    - setStore(fieldName: str, enable: bool = True): Enable/Disable storing a field.
+    - setUpdate(fieldName: str, enable: bool = True): Enable/Disable updating a field.
 
     To exclude a field from the dictionary representation of the instance, set metadata key "exclude" to True.
     To change exclude status of a field, use the following method:
-    - setExclude(field_name: str, enable: bool = True): Exclude a field from dict representation.
+    - setExclude(fieldName: str, enable: bool = True): Exclude a field from dict representation.
     """
 
     ######################
@@ -96,7 +114,7 @@ class DBDataModel:
     ### Conversion methods ###
     ##########################
 
-    def fillDataFromDict(self, kwargs: dict[str, Any]):
+    def fillDataFromDict(self, kwargs: dict[str, Any]) -> None:
         fieldNames = set([f.name for f in dataclasses.fields(self)])
         for key in kwargs:
             if key in fieldNames:
@@ -105,12 +123,25 @@ class DBDataModel:
         self.__post_init__()
 
     # Init data
-    def __post_init__(self):
-        for field_name, field_obj in self.__dataclass_fields__.items():
-            metadata = field_obj.metadata
-            encode = metadata.get("encode", None)
-            if encode is not None:
-                setattr(self, field_name, encode(getattr(self, field_name)))
+    def __post_init__(self) -> None:
+        for fieldName, fieldObj in self.__dataclass_fields__.items():
+            metadata = cast(MetadataDict, fieldObj.metadata)
+            value = getattr(self, fieldName)
+
+            # If serialize is set, and serialize is a SerializeType,
+            # we use our serialization function
+            # Here we actually need to deserialize the value to correct class type
+            serialize = metadata.get("serialize", None)
+            enumClass = metadata.get("enum_class", None)
+            if serialize is not None and isinstance(serialize, SerializeType):
+                value = deserializeValue(value, serialize, enumClass)
+                setattr(self, fieldName, value)
+
+            else:
+                deserialize = metadata.get("deserialize", None)
+                if deserialize is not None:
+                    value = deserialize(value)
+                    setattr(self, fieldName, value)
 
     # String - representation
     def __repr__(self) -> str:
@@ -125,7 +156,7 @@ class DBDataModel:
         for field in pairs:
             classField = self.__dataclass_fields__.get(field[0], None)
             if classField is not None:
-                metadata = classField.metadata
+                metadata = cast(MetadataDict, classField.metadata)
                 if not "exclude" in metadata or not metadata["exclude"]:
                     newDict[field[0]] = field[1]
 
@@ -145,32 +176,20 @@ class DBDataModel:
                 "id": {"type": "number"},
             },
         }
-        for field_name, field_obj in self.__dataclass_fields__.items():
-            metadata = field_obj.metadata
+        for fieldName, fieldObj in self.__dataclass_fields__.items():
+            metadata = cast(MetadataDict, fieldObj.metadata)
             assert (
                 "db_field" in metadata
                 and isinstance(metadata["db_field"], tuple)
                 and len(metadata["db_field"]) == 2
-            ), f"db_field metadata is not set for {field_name}"
+            ), f"db_field metadata is not set for {fieldName}"
             fieldType: str = metadata["db_field"][1]
-            schema["properties"][field_name] = {"type": fieldType}
+            schema["properties"][fieldName] = {"type": fieldType}
 
         return schema
 
     def jsonEncoder(self, obj: Any) -> Any:
-        if isinstance(obj, Decimal):
-            return float(obj)
-
-        if isinstance(obj, datetime.date) or isinstance(obj, datetime.datetime):
-            return obj.strftime("%Y-%m-%dT%H:%M:%S")
-
-        if isinstance(obj, Enum):
-            return obj.value
-
-        if isinstance(obj, int) or isinstance(obj, float) or isinstance(obj, str):
-            return obj
-
-        return str(obj)
+        return jsonEncoder(obj)
 
     def toJsonString(self, pretty: bool = False) -> str:
         if pretty:
@@ -227,35 +246,47 @@ class DBDataModel:
 
         return 0
 
-    def validate(self) -> bool:
+    def validate(self) -> Literal[True] | str:
+        """
+        True if the instance is valid, otherwise an error message.
+        """
         raise NotImplementedError("`validate` is not implemented")
 
-    def setStore(self, field_name: str, enable: bool = True) -> None:
+    def setStore(self, fieldName: str, enable: bool = True) -> None:
         """
         Enable/Disable storing a field (insert into database)
         """
-        if field_name in self.__dataclass_fields__:
-            currentMetadata = self.__dataclass_fields__[field_name].metadata
+        if fieldName in self.__dataclass_fields__:
+            currentMetadata = cast(
+                MetadataDict,
+                dict(self.__dataclass_fields__[fieldName].metadata),
+            )
             currentMetadata["store"] = enable
-            self.__dataclass_fields__[field_name].metadata = currentMetadata
+            self.__dataclass_fields__[fieldName].metadata = currentMetadata
 
-    def setUpdate(self, field_name: str, enable: bool = True) -> None:
+    def setUpdate(self, fieldName: str, enable: bool = True) -> None:
         """
         Enable/Disable updating a field (update in database)
         """
-        if field_name in self.__dataclass_fields__:
-            currentMetadata = self.__dataclass_fields__[field_name].metadata
+        if fieldName in self.__dataclass_fields__:
+            currentMetadata = cast(
+                MetadataDict,
+                dict(self.__dataclass_fields__[fieldName].metadata),
+            )
             currentMetadata["update"] = enable
-            self.__dataclass_fields__[field_name].metadata = currentMetadata
+            self.__dataclass_fields__[fieldName].metadata = currentMetadata
 
-    def setExclude(self, field_name: str, enable: bool = True) -> None:
+    def setExclude(self, fieldName: str, enable: bool = True) -> None:
         """
         Exclude a field from dict representation
         """
-        if field_name in self.__dataclass_fields__:
-            currentMetadata = dict(self.__dataclass_fields__[field_name].metadata)
+        if fieldName in self.__dataclass_fields__:
+            currentMetadata = cast(
+                MetadataDict,
+                dict(self.__dataclass_fields__[fieldName].metadata),
+            )
             currentMetadata["exclude"] = enable
-            self.__dataclass_fields__[field_name].metadata = currentMetadata
+            self.__dataclass_fields__[fieldName].metadata = currentMetadata
 
     ########################
     ### Database methods ###
@@ -272,13 +303,23 @@ class DBDataModel:
         Store data to database
         """
         storeData: dict[str, Any] = {}
-        for field_name, field_obj in self.__dataclass_fields__.items():
-            metadata = field_obj.metadata
+        for fieldName, fieldObj in self.__dataclass_fields__.items():
+            metadata = cast(MetadataDict, fieldObj.metadata)
             if "store" in metadata and metadata["store"] == True:
-                storeData[field_name] = getattr(self, field_name)
-
-                if "decode" in metadata and metadata["decode"] is not None:
-                    storeData[field_name] = metadata["decode"](storeData[field_name])
+                storeData[fieldName] = getattr(self, fieldName)
+
+                # If serialize is set, and serialize is a SerializeType,
+                # we use our serialization function.
+                # Otherwise, we use the provided serialize function
+                # and we assume that it is callable
+                serialize = metadata.get("serialize", None)
+                if serialize is not None:
+                    if isinstance(serialize, SerializeType):
+                        storeData[fieldName] = serializeValue(
+                            storeData[fieldName], serialize
+                        )
+                    else:
+                        storeData[fieldName] = serialize(storeData[fieldName])
 
         return storeData
 
@@ -288,13 +329,23 @@ class DBDataModel:
         """
 
         updateData: dict[str, Any] = {}
-        for field_name, field_obj in self.__dataclass_fields__.items():
-            metadata = field_obj.metadata
+        for fieldName, fieldObj in self.__dataclass_fields__.items():
+            metadata = cast(MetadataDict, fieldObj.metadata)
             if "update" in metadata and metadata["update"] == True:
-                updateData[field_name] = getattr(self, field_name)
-
-                if "decode" in metadata and metadata["decode"] is not None:
-                    updateData[field_name] = metadata["decode"](updateData[field_name])
+                updateData[fieldName] = getattr(self, fieldName)
+
+                # If serialize is set, and serialize is a SerializeType,
+                # we use our serialization function.
+                # Otherwise, we use the provided serialize function
+                # and we assume that it is callable
+                serialize = metadata.get("serialize", None)
+                if serialize is not None:
+                    if isinstance(serialize, SerializeType):
+                        updateData[fieldName] = serializeValue(
+                            updateData[fieldName], serialize
+                        )
+                    else:
+                        updateData[fieldName] = serialize(updateData[fieldName])
 
         return updateData
 
@@ -321,8 +372,7 @@ class DBDefaultsDataModel(DBDataModel):
             "db_field": ("created_at", "timestamptz"),
             "store": True,
             "update": False,
-            "encode": lambda value: DBDataModel.strToDatetime(value),  # type: ignore
-            "decode": lambda x: x.isoformat(),  # type: ignore
+            "serialize": SerializeType.DATETIME,
         },
     )
     """created_at is readonly by default and should be present in all tables"""
@@ -333,8 +383,7 @@ class DBDefaultsDataModel(DBDataModel):
             "db_field": ("updated_at", "timestamptz"),
             "store": True,
             "update": True,
-            "encode": lambda value: DBDataModel.strToDatetime(value),  # type: ignore
-            "decode": lambda x: x.isoformat(),  # type: ignore
+            "serialize": SerializeType.DATETIME,
         },
     )
     """updated_at is readonly by default and should be present in all tables"""