openmodule 13.3.0__tar.gz → 13.5.0__tar.gz

{openmodule-13.3.0 → openmodule-13.5.0}/AUTHORS

@@ -1,6 +1,7 @@
 Kevin Koller <k.koller@accessio.at>
 Kevin Koller <k.koller@arivo.co>
 Matthias Mietschnig <m.mietschnig@arivo.co>
+Maximilian Bialek <m.bialek@arivo.co>
 Philipp Reitter <i@philipp.ninja>
 Philipp Reitter <p.reitter@accessio.at>
 Philipp Reitter <p.reitter@gmail.com>

{openmodule-13.3.0 → openmodule-13.5.0}/ChangeLog

@@ -1,6 +1,17 @@
 CHANGES
 =======
 
+v13.5.0
+-------
+
+* document breaking changes
+* OM-405: remove database imports from non-database services + isolate migration process to exclude alembic import in the main process
+
+v13.4.0
+-------
+
+* Om 164: settings-models update for enforcment settings changes
+
 v13.3.0
 -------
 
@@ -11,6 +22,7 @@ v13.2.0
 
 * OM 187 Alembic migration test mixin
 * add a testcase for expiring session data while adding new models (noticed in OM-327)
+* OM-184 Openmodule Database: Migration Backups and Fix in Operations "pre\_downgrade()" and "post\_downgrade()"
 * Aktualisierung der Dokumentation
 * OM-255
 
@@ -152,12 +164,6 @@ v11.1.0.rc5
 -----------
 
 * again no language set in testing
-
-v11.1.0.rc4
------------
-
-* arivo-schedule in public pip
-* use temporary arivo-schedule package
 * removed mock rpcs from schema
 
 v11.0.3
@@ -209,12 +215,3 @@ v10.0.2.rc1
 -----------
 
 * Refactor for backend class for controller v2
-
-v10.0.1
--------
-
-* fix for ondelete=CASCADE database migration problem
-
-v10.0.0
--------
-

{openmodule-13.3.0/openmodule.egg-info → openmodule-13.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openmodule
-Version: 13.3.0
+Version: 13.5.0
 Summary: Libraries for developing the arivo openmodule
 Home-page: https://gitlab.com/arivo-public/device-python/openmodule.git
 Author: ARIVO

{openmodule-13.3.0 → openmodule-13.5.0}/docs/migrations.md

@@ -1,5 +1,11 @@
 # Breaking Version Changes
 
+## 13.5.0:
+* changes in env.py of alembic: <br>
+  `from openmodule.database.database import run_env_py` -> <br> 
+  `from openmodule.database.migration import run_env_py`
+
+
 ## 13.0.0
 * access_service:
   * changes in AccessCheckAccess (removed media field) and AccessCheckResponse (removed error field)

{openmodule-13.3.0 → openmodule-13.5.0}/openmodule/core.py

@@ -11,7 +11,6 @@ from typing import Optional
 
 from openmodule.alert import AlertHandler
 from openmodule.config import validate_config_module
-from openmodule.database.database import Database
 from openmodule.dispatcher import ZMQMessageDispatcher
 from openmodule.health import HealthHandlerType, Healthz, HealthPingMessage
 from openmodule.logging import init_logging
@@ -23,7 +22,7 @@ from openmodule.connection_status import ConnectionStatusListener
 
 
 class OpenModuleCore(threading.Thread):
-    database: Optional[Database] = None
+    database = None
 
     def __init__(self, context, config, messages_executor=None):
         super().__init__(target=get_thread_wrapper(self._run))
@@ -54,6 +53,7 @@ class OpenModuleCore(threading.Thread):
                                                  match_type=True, register_schema=False)
 
     def init_database(self):
+        from openmodule.database.database import Database
         if self.config.TESTING:
             self.database = Database(self.config.DATABASE_FOLDER, self.config.NAME, alembic_path="../src/database")
         else:

openmodule-13.5.0/openmodule/database/database.py

@@ -0,0 +1,169 @@
+import os
+import threading
+import types
+from typing import Optional, Tuple
+
+from sqlalchemy import create_engine, event
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import sessionmaker, Session
+from sqlalchemy.pool import StaticPool
+
+from multiprocessing import Process, Pipe, ProcessError
+import traceback
+
+
+class MigrationError(BaseException):
+    # Exception wrapper to create exception with traceback from child process
+    pass
+
+
+class MigrationProcess(Process):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._parent_conn, self._child_conn = Pipe()
+        self._exception: Optional[Tuple[Exception, str]] = None
+
+    def run(self):
+        try:
+            super().run()
+            self._child_conn.send(None)
+        except Exception as e:
+            tb = traceback.format_exc()
+            self._child_conn.send((e, tb))
+
+    @property
+    def exception(self) -> Optional[Tuple[Exception, str]]:
+        if self._parent_conn.poll():
+            self._exception = self._parent_conn.recv()
+        return self._exception
+
+
+@event.listens_for(Engine, "connect")
+def set_sqlite_pragma(dbapi_connection, connection_record):
+    cursor = dbapi_connection.cursor()
+    cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.close()
+
+
+active_databases = {}
+
+
+def execute_migration(engine: Engine, alembic_path: Optional[str] = None):
+    def isolated_migration_process():
+        nonlocal engine, alembic_path
+        from openmodule.database.migration import migrate_database
+        migrate_database(engine, alembic_path)
+
+    p = MigrationProcess(target=isolated_migration_process)
+    try:
+        p.start()
+        p.join(timeout=5 * 60)  # default 5 min timeout -> exception/sentry in service
+    except ProcessError:
+        raise
+    finally:
+        if p.is_alive():
+            p.kill()  # send SIGKILL
+            p.join()  # wait for death
+            p.close()  # release all resources associated with the process
+
+    if p.exitcode is None:
+        raise TimeoutError("The duration of the migration exceeded the timeout")
+
+    if p.exception is not None:
+        # piped exception with full traceback from child process -> if migration failed e.g. bad migration file
+        e, tb = p.exception
+        raise MigrationError(tb) from e
+
+
+def database_path(db_folder, db_name):
+    return os.path.join(db_folder, db_name) + ".sqlite3"
+
+
+def get_database(db_folder: str, name: str, alembic_path: Optional[str] = None):
+    global active_databases
+    tmp = database_path(db_folder, name)
+    assert active_databases.get(tmp) is None, f"database {tmp} already exists," \
+                                              f" check if it was shutdown before a new one was created"
+    os.makedirs(db_folder, exist_ok=True)
+    path = f"sqlite:///{tmp}"
+    engine = create_engine(path, poolclass=StaticPool, connect_args={'check_same_thread': False})
+
+    # migration executed in a separate process -> no alembic import in main process
+    execute_migration(engine, alembic_path)
+
+    active_databases[tmp] = engine
+
+    return engine
+
+
+class DatabaseContext:
+    def __init__(self, database: 'Database', expire_on_commit=True):
+        self.database = database
+        self.expire_on_commit = expire_on_commit
+
+    def __enter__(self) -> Session:
+        return self.database.__enter__(expire_on_commit=self.expire_on_commit)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return self.database.__exit__(exc_type, exc_val, exc_tb)
+
+
+class SessionWrapper(Session):
+    _closed = False
+
+    def __getattribute__(self, item):
+        attr = super().__getattribute__(item)
+        if isinstance(attr, types.MethodType) and self._closed:
+            raise AssertionError("Session is already closed")
+        return attr
+
+    def close(self):
+        super().close()
+        self._closed = True
+
+
+class Database:
+    active_session: Optional[Session]
+
+    def __init__(self, database_folder, name="database", alembic_path=None):
+        self.db_folder = database_folder
+        self.name = name
+        self._engine = get_database(database_folder, name, alembic_path)
+        self._session = sessionmaker(bind=self._engine, class_=SessionWrapper)
+        self.active_session = None
+        self.lock = threading.RLock()
+
+    def is_open(self):
+        return bool(self._session)
+
+    def shutdown(self):
+        assert self.is_open(), "database is already closed, you called shutdown twice somewhere"
+
+        with self.lock:
+            self._session = None
+            global active_databases
+            active_databases.pop(database_path(self.db_folder, self.name), None)
+
+    def __call__(self, expire_on_commit=True) -> DatabaseContext:
+        return DatabaseContext(self, expire_on_commit=expire_on_commit)
+
+    def __enter__(self, expire_on_commit=True) -> Session:
+        assert self._session, "Session is already closed"
+        self.lock.acquire()
+        self.active_session = self._session(expire_on_commit=expire_on_commit)
+        return self.active_session
+
+    def flush(self, objects=None):
+        assert self._session and self.active_session, "Session is already closed"
+        self.active_session.flush(objects)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        try:
+            if exc_type is None:
+                self.active_session.commit()
+            else:
+                self.active_session.rollback()
+        finally:
+            self.active_session.close()
+            self.active_session = None
+            self.lock.release()

{openmodule-13.3.0 → openmodule-13.5.0}/openmodule/database/env.py

@@ -4,7 +4,6 @@ from alembic import context
 from sqlalchemy import engine_from_config
 from sqlalchemy import pool
 
-from openmodule.checks import check_invalid_database_column_type
 from openmodule.database.custom_types import CustomType
 
 # this is the Alembic Config object, which provides

openmodule-13.3.0/openmodule/database/database.py → openmodule-13.5.0/openmodule/database/migration.py

@@ -2,39 +2,18 @@ import datetime
 import os
 import shutil
 import sys
-import threading
-import types
 import warnings
 from typing import Optional
-
+  
 from alembic import command, context
 from alembic.autogenerate import comparators, renderers
 from alembic.config import Config
 from alembic.operations import Operations, MigrateOperation
 from alembic.runtime.migration import MigrationContext
-from sqlalchemy import create_engine, event, MetaData
+
+from sqlalchemy import MetaData, DateTime
 from sqlalchemy.engine import Engine
 from sqlalchemy.engine.reflection import Inspector
-from sqlalchemy.orm import sessionmaker, Session
-from sqlalchemy.pool import StaticPool
-
-from openmodule.checks import check_invalid_database_column_type
-
-
-def drop_alembic_tmp_tables(op):
-    conn = op.get_bind()
-    inspector = Inspector.from_engine(conn)
-    tables = inspector.get_table_names()
-    for table in tables:
-        if table.startswith("_alembic_tmp_"):
-            op.drop_table(table)
-
-
-@event.listens_for(Engine, "connect")
-def set_sqlite_pragma(dbapi_connection, connection_record):
-    cursor = dbapi_connection.cursor()
-    cursor.execute("PRAGMA foreign_keys=ON")
-    cursor.close()
 
 
 @Operations.register_operation("pre_upgrade")
@@ -43,7 +22,9 @@ class PreUpgradeOp(MigrateOperation):
     def pre_upgrade(cls, operations, **kw):
         migration_context: MigrationContext = operations.migration_context
         basename, _ = os.path.splitext(os.path.basename(migration_context.connection.engine.url.database))
-        filename = f"{basename}_{datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')}.sqlite3.backup"
+        timestamp = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')
+        migration_revision = migration_context.get_current_revision()
+        filename = f"{basename}_{timestamp}_{migration_revision}.sqlite3.backup"
 
         shutil.copy(migration_context.connection.engine.url.database,
                     os.path.join(os.path.dirname(migration_context.connection.engine.url.database), filename))
@@ -96,7 +77,12 @@ class PostDowngradeOp(MigrateOperation):
 def pre_upgrade(operations, operation):
     # NOTE: This is currently in sync with pre_downgrade, if you want to have
     # different behavior, you'll need to change th pre_downgrade function below
-    drop_alembic_tmp_tables(operations)
+    conn = operations.get_bind()
+    inspector = Inspector.from_engine(conn)
+    tables = inspector.get_table_names()
+    for table in tables:
+        if table.startswith("_alembic_tmp_"):
+            operations.drop_table(table)
     operations.execute("PRAGMA foreign_keys = OFF")
 
 
@@ -143,7 +129,7 @@ def add_pre_upgrade_hooks(autogen_context, upgrade_ops, schemas):
         upgrade_ops.ops.append(PostUpgradeOp())
 
 
-def alembic_config(connection, alembic_path):
+def alembic_config(connection: Engine, alembic_path: str):
     alembic_cfg = Config(os.path.join(alembic_path, "alembic.ini"),
                          attributes={
                              "configure_logging": False,
@@ -153,16 +139,15 @@ def alembic_config(connection, alembic_path):
     return alembic_cfg
 
 
-def migrate_database(connection, alembic_path=None):
+def migrate_database(connection: Engine, alembic_path: Optional[str] = None):
     if alembic_path is None:
         alembic_path = os.path.join(os.getcwd(), "database")
     assert os.path.exists(os.path.abspath(alembic_path)), f"alembic path {os.path.abspath(alembic_path)} does not exist"
     config = alembic_config(connection, alembic_path)
     command.upgrade(config, "head")
-    assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1, "foreign keys are not enabled"
 
-
-active_databases = {}
+    check = connection.execute("PRAGMA foreign_keys").fetchone()
+    assert check is not None and check[0] == 1, "foreign keys are not enabled"
 
 
 def register_bases(bases, show_deprecation_warning=True):
@@ -193,92 +178,7 @@ def run_env_py(bases):
     del sys.modules["openmodule.database.env"]  # unload the module, so we can re-run it (mostly testcases)
 
 
-def database_path(db_folder, db_name):
-    return os.path.join(db_folder, db_name) + ".sqlite3"
-
-
-def get_database(db_folder: str, name: str, alembic_path=None):
-    global active_databases
-    tmp = database_path(db_folder, name)
-    assert active_databases.get(tmp) is None, f"database {tmp} already exists," \
-                                              f" check if it was shutdown before a new one was created"
-    os.makedirs(db_folder, exist_ok=True)
-    path = f"sqlite:///{tmp}"
-    engine = create_engine(path, poolclass=StaticPool, connect_args={'check_same_thread': False})
-    migrate_database(engine, alembic_path)
-    active_databases[tmp] = engine
-
-    return engine
-
-
-class DatabaseContext:
-    def __init__(self, database: 'Database', expire_on_commit=True):
-        self.database = database
-        self.expire_on_commit = expire_on_commit
-
-    def __enter__(self) -> Session:
-        return self.database.__enter__(expire_on_commit=self.expire_on_commit)
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        return self.database.__exit__(exc_type, exc_val, exc_tb)
-
-
-class SessionWrapper(Session):
-    _closed = False
-
-    def __getattribute__(self, item):
-        attr = super().__getattribute__(item)
-        if isinstance(attr, types.MethodType) and self._closed:
-            raise AssertionError("Session is already closed")
-        return attr
-
-    def close(self):
-        super().close()
-        self._closed = True
-
-
-class Database:
-    active_session: Optional[Session]
-
-    def __init__(self, database_folder, name="database", alembic_path=None):
-        self.db_folder = database_folder
-        self.name = name
-        self._engine = get_database(database_folder, name, alembic_path)
-        self._session = sessionmaker(bind=self._engine, class_=SessionWrapper)
-        self.active_session = None
-        self.lock = threading.RLock()
-
-    def is_open(self):
-        return bool(self._session)
-
-    def shutdown(self):
-        assert self.is_open(), "database is already closed, you called shutdown twice somewhere"
-
-        with self.lock:
-            self._session = None
-            global active_databases
-            active_databases.pop(database_path(self.db_folder, self.name), None)
-
-    def __call__(self, expire_on_commit=True) -> DatabaseContext:
-        return DatabaseContext(self, expire_on_commit=expire_on_commit)
-
-    def __enter__(self, expire_on_commit=True) -> Session:
-        assert self._session, "Session is already closed"
-        self.lock.acquire()
-        self.active_session = self._session(expire_on_commit=expire_on_commit)
-        return self.active_session
-
-    def flush(self, objects=None):
-        assert self._session and self.active_session, "Session is already closed"
-        self.active_session.flush(objects)
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        try:
-            if exc_type is None:
-                self.active_session.commit()
-            else:
-                self.active_session.rollback()
-        finally:
-            self.active_session.close()
-            self.active_session = None
-            self.lock.release()
+def check_invalid_database_column_type(typ):
+    from openmodule import config
+    if config.run_checks():
+        assert not isinstance(typ, DateTime), "Do NOT use DateTime fields, use TZDateTime fields instead"

{openmodule-13.3.0 → openmodule-13.5.0}/openmodule/models/base.py

@@ -22,12 +22,27 @@ def _donotuse(v, *, default):
 
 
 if config.run_checks():
-    from openmodule import checks
-
-    meta_kwargs = {"metaclass": checks.CheckingOpenModuleModel}
+    class CheckingOpenModuleModel(type(BaseModel), type):
+        def __new__(mcs, name, bases, dct):
+            cls = super().__new__(mcs, name, bases, dct)
+
+            # noinspection PyUnresolvedReferences
+            for field_name, field in cls.__fields__.items():
+                if field.type_ == datetime:
+                    assert "_timezone_validator" in field.class_validators, (
+                        "datetime fields must use the timezone_validator to ensure all datetime values "
+                        "are always naive. Otherwise runtime errors may occur depending on the isoformat used.\n"
+                        f"In class {name}, please add:\n"
+                        f'  _tz_{field_name} = timezone_validator("{field_name}")'
+                    )
+
+            return cls
+
+    meta_kwargs = {"metaclass": CheckingOpenModuleModel}
 else:
     meta_kwargs = {}
 
+
 ESCAPE_ASCII = re.compile(r'([^ -~])')
 
 

{openmodule-13.3.0 → openmodule-13.5.0/openmodule.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openmodule
-Version: 13.3.0
+Version: 13.5.0
 Summary: Libraries for developing the arivo openmodule
 Home-page: https://gitlab.com/arivo-public/device-python/openmodule.git
 Author: ARIVO

{openmodule-13.3.0 → openmodule-13.5.0}/openmodule.egg-info/SOURCES.txt

@@ -43,7 +43,6 @@ docs/deprecated_code/package_reader/tests/test_package_reader.py
 docs/images/broker.drawio.png
 openmodule/__init__.py
 openmodule/alert.py
-openmodule/checks.py
 openmodule/config.py
 openmodule/connection_status.py
 openmodule/core.py
@@ -63,6 +62,7 @@ openmodule.egg-info/top_level.txt
 openmodule/database/custom_types.py
 openmodule/database/database.py
 openmodule/database/env.py
+openmodule/database/migration.py
 openmodule/models/__init__.py
 openmodule/models/access_service.py
 openmodule/models/alert.py
@@ -167,6 +167,20 @@ tests/invalid_database/alembic/__init__.py
 tests/invalid_database/alembic/env.py
 tests/invalid_database/alembic/script.py.mako
 tests/invalid_database/alembic/versions/ff26e54332f9_datetime_models.py
+tests/migration_double_column_delete_error/alembic.ini
+tests/migration_double_column_delete_error/makemigration.sh
+tests/migration_double_column_delete_error/alembic/__init__.py
+tests/migration_double_column_delete_error/alembic/env.py
+tests/migration_double_column_delete_error/alembic/script.py.mako
+tests/migration_double_column_delete_error/alembic/versions/812a3e5b8517_initial.py
+tests/migration_double_column_delete_error/alembic/versions/a7ea100a784f_key_error.py
+tests/migration_no_such_table_error/alembic.ini
+tests/migration_no_such_table_error/makemigration.sh
+tests/migration_no_such_table_error/alembic/__init__.py
+tests/migration_no_such_table_error/alembic/env.py
+tests/migration_no_such_table_error/alembic/script.py.mako
+tests/migration_no_such_table_error/alembic/versions/812a3e5b8517_initial.py
+tests/migration_no_such_table_error/alembic/versions/a7ea100a784f_no_such_table_error.py
 tests/migration_test_database/__init__.py
 tests/migration_test_database/alembic.ini
 tests/migration_test_database/alembic_migration_test_database.sqlite3

openmodule-13.5.0/openmodule.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "84be0cd", "is_release": true}

{openmodule-13.3.0 → openmodule-13.5.0}/openmodule.egg-info/requires.txt

@@ -10,7 +10,7 @@ requests>=2.22.0
 python-dateutil>=2.7.2
 python-dotenv==0.15.0
 redis<4,>=3.5
-arivo-settings_models<2,>=1.1.3
+arivo-settings_models<2,>=1.1.4
 arivo-schedule~=1.2.2
 
 [commands]

{openmodule-13.3.0 → openmodule-13.5.0}/openmodule_test/database.py

@@ -9,7 +9,10 @@ from sqlalchemy import MetaData
 from sqlalchemy.ext.automap import automap_base
 
 from openmodule.config import settings
-from openmodule.database.database import Database, database_path, alembic_config
+from openmodule.database.database import Database, database_path
+from openmodule.database.migration import alembic_config
+
+from unittest.mock import patch
 
 _first_start = True
 
@@ -37,6 +40,7 @@ class SQLiteTestMixin(TestCase):
     database_folder: str = None  # defaults to settings.DATABASE_FOLDER
     alembic_path = "../src/database"
     database_name = "database"
+    main_process_migration = True  # change if migration should be performed in a separate process
 
     @classmethod
     def get_database_folder(cls):
@@ -44,6 +48,15 @@ class SQLiteTestMixin(TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
+        if cls.main_process_migration is True:
+            from openmodule.database.migration import migrate_database
+            # instead of migrating db in a child process we 'mock' it in the main process to prevent some errors
+            cls.patcher = patch(
+                'openmodule.database.database.execute_migration',
+                new=lambda x, y: migrate_database(x, y)
+            )
+            cls.patcher.start()
+
         # we only know which databases are in use on tear down, so truncating only works in teardown
         # but in order to not be annoyed by failed tests which left broken databases, we delete all databases
         # once initially
@@ -75,6 +88,8 @@ class SQLiteTestMixin(TestCase):
             cls.database.shutdown()
             os.unlink(database_path(cls.get_database_folder(), cls.database_name))
         super().tearDownClass()
+        if cls.main_process_migration is True:
+            cls.patcher.stop()
 
 
 class AlembicMigrationTestMixin(SQLiteTestMixin):

{openmodule-13.3.0 → openmodule-13.5.0}/requirements.txt

@@ -10,5 +10,5 @@ requests>=2.22.0
 python-dateutil>=2.7.2
 python-dotenv==0.15.0
 redis>=3.5,<4
-arivo-settings_models>=1.1.3,<2
+arivo-settings_models>=1.1.4,<2
 arivo-schedule~=1.2.2

{openmodule-13.3.0 → openmodule-13.5.0}/tests/invalid_database/alembic/env.py

@@ -1,7 +1,7 @@
 from sqlalchemy import Column, Integer, DateTime
 from sqlalchemy.ext.declarative import declarative_base
 
-from openmodule.database.database import run_env_py
+from openmodule.database.migration import run_env_py
 
 Base = declarative_base()
 

{openmodule-13.3.0/tests/test_access_service_database → openmodule-13.5.0/tests/migration_double_column_delete_error}/alembic/env.py

@@ -1,4 +1,4 @@
-from openmodule.database.database import run_env_py
+from openmodule.database.migration import run_env_py
 from tests.test_utils_access_service import Base
 
 run_env_py([Base])