openmodule 13.4.0__tar.gz → 13.6.0__tar.gz

{openmodule-13.4.0 → openmodule-13.6.0}/AUTHORS

@@ -1,6 +1,7 @@
 Kevin Koller <k.koller@accessio.at>
 Kevin Koller <k.koller@arivo.co>
 Matthias Mietschnig <m.mietschnig@arivo.co>
+Maximilian Bialek <m.bialek@arivo.co>
 Philipp Reitter <i@philipp.ninja>
 Philipp Reitter <p.reitter@accessio.at>
 Philipp Reitter <p.reitter@gmail.com>

{openmodule-13.4.0 → openmodule-13.6.0}/ChangeLog

@@ -1,6 +1,17 @@
 CHANGES
 =======
 
+v13.6.0
+-------
+
+* OM-697 OpenModule Verbesserungen
+
+v13.5.0
+-------
+
+* document breaking changes
+* OM-405: remove database imports from non-database services + isolate migration process to exclude alembic import in the main process
+
 v13.4.0
 -------
 
@@ -16,6 +27,7 @@ v13.2.0
 
 * OM 187 Alembic migration test mixin
 * add a testcase for expiring session data while adding new models (noticed in OM-327)
+* OM-184 Openmodule Database: Migration Backups and Fix in Operations "pre\_downgrade()" and "post\_downgrade()"
 * Aktualisierung der Dokumentation
 * OM-255
 
@@ -152,16 +164,6 @@ v11.1.0
 -------
 
 * csv export library, databox upload and schedule library tips
-
-v11.1.0.rc5
------------
-
-* again no language set in testing
-
-v11.1.0.rc4
------------
-
-* arivo-schedule in public pip
 * removed mock rpcs from schema
 
 v11.0.3
@@ -208,12 +210,3 @@ v10.0.2.rc2
 
 * added more documentation [skip ci]
 * fixes an issue in the multiprocessing\_logging package in testcases
-
-v10.0.2.rc1
------------
-
-* Refactor for backend class for controller v2
-
-v10.0.1
--------
-

{openmodule-13.4.0/openmodule.egg-info → openmodule-13.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openmodule
-Version: 13.4.0
+Version: 13.6.0
 Summary: Libraries for developing the arivo openmodule
 Home-page: https://gitlab.com/arivo-public/device-python/openmodule.git
 Author: ARIVO

{openmodule-13.4.0 → openmodule-13.6.0}/docs/migrations.md

@@ -1,5 +1,18 @@
 # Breaking Version Changes
 
+## 13.6.0
+* messaging:
+  * the `get_pub_socket` now raises an AssertionError because we no longer support multiple publishers. 
+    This was done because pub sockets receive all subscriptions and queue them until the next message is sent.
+    This can lead to a memory leak.
+* logging:
+  * logs will now be written to stdout instead of stderr
+
+## 13.5.0:
+* changes in env.py of alembic: <br>
+  `from openmodule.database.database import run_env_py` -> <br> 
+  `from openmodule.database.migration import run_env_py`
+
 ## 13.0.0
 * access_service:
   * changes in AccessCheckAccess (removed media field) and AccessCheckResponse (removed error field)

{openmodule-13.4.0 → openmodule-13.6.0}/openmodule/core.py

@@ -11,11 +11,11 @@ from typing import Optional
 
 from openmodule.alert import AlertHandler
 from openmodule.config import validate_config_module
-from openmodule.database.database import Database
 from openmodule.dispatcher import ZMQMessageDispatcher
 from openmodule.health import HealthHandlerType, Healthz, HealthPingMessage
 from openmodule.logging import init_logging
-from openmodule.messaging import get_pub_socket, get_sub_socket, receive_message_from_socket, wait_for_connection
+from openmodule.messaging import _internal_get_pub_socket, get_sub_socket, receive_message_from_socket, \
+    wait_for_connection
 from openmodule.models.base import ZMQMessage
 from openmodule.sentry import init_sentry, should_activate_sentry, deinit_sentry
 from openmodule.threading import get_thread_wrapper
@@ -23,7 +23,7 @@ from openmodule.connection_status import ConnectionStatusListener
 
 
 class OpenModuleCore(threading.Thread):
-    database: Optional[Database] = None
+    database = None
 
     def __init__(self, context, config, messages_executor=None):
         super().__init__(target=get_thread_wrapper(self._run))
@@ -36,7 +36,7 @@ class OpenModuleCore(threading.Thread):
         self.pub_lock = threading.Lock()
         self.sub_lock = threading.Lock()
 
-        self.pub_socket = get_pub_socket(self.context, self.config, linger=1000)
+        self.pub_socket = _internal_get_pub_socket(self.context, self.config, linger=1000)
         self.sub_socket = get_sub_socket(self.context, self.config)
         self.sub_socket_internal = get_sub_socket(self.context, self.config)
 
@@ -54,6 +54,7 @@ class OpenModuleCore(threading.Thread):
                                                  match_type=True, register_schema=False)
 
     def init_database(self):
+        from openmodule.database.database import Database
         if self.config.TESTING:
             self.database = Database(self.config.DATABASE_FOLDER, self.config.NAME, alembic_path="../src/database")
         else:

openmodule-13.6.0/openmodule/database/database.py

@@ -0,0 +1,169 @@
+import os
+import threading
+import types
+from typing import Optional, Tuple
+
+from sqlalchemy import create_engine, event
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import sessionmaker, Session
+from sqlalchemy.pool import StaticPool
+
+from multiprocessing import Process, Pipe, ProcessError
+import traceback
+
+
+class MigrationError(BaseException):
+    # Exception wrapper to create exception with traceback from child process
+    pass
+
+
+class MigrationProcess(Process):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._parent_conn, self._child_conn = Pipe()
+        self._exception: Optional[Tuple[Exception, str]] = None
+
+    def run(self):
+        try:
+            super().run()
+            self._child_conn.send(None)
+        except Exception as e:
+            tb = traceback.format_exc()
+            self._child_conn.send((e, tb))
+
+    @property
+    def exception(self) -> Optional[Tuple[Exception, str]]:
+        if self._parent_conn.poll():
+            self._exception = self._parent_conn.recv()
+        return self._exception
+
+
+@event.listens_for(Engine, "connect")
+def set_sqlite_pragma(dbapi_connection, connection_record):
+    cursor = dbapi_connection.cursor()
+    cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.close()
+
+
+active_databases = {}
+
+
+def execute_migration(engine: Engine, alembic_path: Optional[str] = None):
+    def isolated_migration_process():
+        nonlocal engine, alembic_path
+        from openmodule.database.migration import migrate_database
+        migrate_database(engine, alembic_path)
+
+    p = MigrationProcess(target=isolated_migration_process)
+    try:
+        p.start()
+        p.join(timeout=5 * 60)  # default 5 min timeout -> exception/sentry in service
+    except ProcessError:
+        raise
+    finally:
+        if p.is_alive():
+            p.kill()  # send SIGKILL
+            p.join()  # wait for death
+            p.close()  # release all resources associated with the process
+
+    if p.exitcode is None:
+        raise TimeoutError("The duration of the migration exceeded the timeout")
+
+    if p.exception is not None:
+        # piped exception with full traceback from child process -> if migration failed e.g. bad migration file
+        e, tb = p.exception
+        raise MigrationError(tb) from e
+
+
+def database_path(db_folder, db_name):
+    return os.path.join(db_folder, db_name) + ".sqlite3"
+
+
+def get_database(db_folder: str, name: str, alembic_path: Optional[str] = None):
+    global active_databases
+    tmp = database_path(db_folder, name)
+    assert active_databases.get(tmp) is None, f"database {tmp} already exists," \
+                                              f" check if it was shutdown before a new one was created"
+    os.makedirs(db_folder, exist_ok=True)
+    path = f"sqlite:///{tmp}"
+    engine = create_engine(path, poolclass=StaticPool, connect_args={'check_same_thread': False})
+
+    # migration executed in a separate process -> no alembic import in main process
+    execute_migration(engine, alembic_path)
+
+    active_databases[tmp] = engine
+
+    return engine
+
+
+class DatabaseContext:
+    def __init__(self, database: 'Database', expire_on_commit=True):
+        self.database = database
+        self.expire_on_commit = expire_on_commit
+
+    def __enter__(self) -> Session:
+        return self.database.__enter__(expire_on_commit=self.expire_on_commit)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return self.database.__exit__(exc_type, exc_val, exc_tb)
+
+
+class SessionWrapper(Session):
+    _closed = False
+
+    def __getattribute__(self, item):
+        attr = super().__getattribute__(item)
+        if isinstance(attr, types.MethodType) and self._closed:
+            raise AssertionError("Session is already closed")
+        return attr
+
+    def close(self):
+        super().close()
+        self._closed = True
+
+
+class Database:
+    active_session: Optional[Session]
+
+    def __init__(self, database_folder, name="database", alembic_path=None):
+        self.db_folder = database_folder
+        self.name = name
+        self._engine = get_database(database_folder, name, alembic_path)
+        self._session = sessionmaker(bind=self._engine, class_=SessionWrapper)
+        self.active_session = None
+        self.lock = threading.RLock()
+
+    def is_open(self):
+        return bool(self._session)
+
+    def shutdown(self):
+        assert self.is_open(), "database is already closed, you called shutdown twice somewhere"
+
+        with self.lock:
+            self._session = None
+            global active_databases
+            active_databases.pop(database_path(self.db_folder, self.name), None)
+
+    def __call__(self, expire_on_commit=True) -> DatabaseContext:
+        return DatabaseContext(self, expire_on_commit=expire_on_commit)
+
+    def __enter__(self, expire_on_commit=True) -> Session:
+        assert self._session, "Session is already closed"
+        self.lock.acquire()
+        self.active_session = self._session(expire_on_commit=expire_on_commit)
+        return self.active_session
+
+    def flush(self, objects=None):
+        assert self._session and self.active_session, "Session is already closed"
+        self.active_session.flush(objects)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        try:
+            if exc_type is None:
+                self.active_session.commit()
+            else:
+                self.active_session.rollback()
+        finally:
+            self.active_session.close()
+            self.active_session = None
+            self.lock.release()

{openmodule-13.4.0 → openmodule-13.6.0}/openmodule/database/env.py

@@ -4,7 +4,6 @@ from alembic import context
 from sqlalchemy import engine_from_config
 from sqlalchemy import pool
 
-from openmodule.checks import check_invalid_database_column_type
 from openmodule.database.custom_types import CustomType
 
 # this is the Alembic Config object, which provides

openmodule-13.4.0/openmodule/database/database.py → openmodule-13.6.0/openmodule/database/migration.py

@@ -2,39 +2,18 @@ import datetime
 import os
 import shutil
 import sys
-import threading
-import types
 import warnings
 from typing import Optional
-
+  
 from alembic import command, context
 from alembic.autogenerate import comparators, renderers
 from alembic.config import Config
 from alembic.operations import Operations, MigrateOperation
 from alembic.runtime.migration import MigrationContext
-from sqlalchemy import create_engine, event, MetaData
+
+from sqlalchemy import MetaData, DateTime
 from sqlalchemy.engine import Engine
 from sqlalchemy.engine.reflection import Inspector
-from sqlalchemy.orm import sessionmaker, Session
-from sqlalchemy.pool import StaticPool
-
-from openmodule.checks import check_invalid_database_column_type
-
-
-def drop_alembic_tmp_tables(op):
-    conn = op.get_bind()
-    inspector = Inspector.from_engine(conn)
-    tables = inspector.get_table_names()
-    for table in tables:
-        if table.startswith("_alembic_tmp_"):
-            op.drop_table(table)
-
-
-@event.listens_for(Engine, "connect")
-def set_sqlite_pragma(dbapi_connection, connection_record):
-    cursor = dbapi_connection.cursor()
-    cursor.execute("PRAGMA foreign_keys=ON")
-    cursor.close()
 
 
 @Operations.register_operation("pre_upgrade")
@@ -43,7 +22,9 @@ class PreUpgradeOp(MigrateOperation):
     def pre_upgrade(cls, operations, **kw):
         migration_context: MigrationContext = operations.migration_context
         basename, _ = os.path.splitext(os.path.basename(migration_context.connection.engine.url.database))
-        filename = f"{basename}_{datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')}.sqlite3.backup"
+        timestamp = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')
+        migration_revision = migration_context.get_current_revision()
+        filename = f"{basename}_{timestamp}_{migration_revision}.sqlite3.backup"
 
         shutil.copy(migration_context.connection.engine.url.database,
                     os.path.join(os.path.dirname(migration_context.connection.engine.url.database), filename))
@@ -96,7 +77,12 @@ class PostDowngradeOp(MigrateOperation):
 def pre_upgrade(operations, operation):
     # NOTE: This is currently in sync with pre_downgrade, if you want to have
     # different behavior, you'll need to change th pre_downgrade function below
-    drop_alembic_tmp_tables(operations)
+    conn = operations.get_bind()
+    inspector = Inspector.from_engine(conn)
+    tables = inspector.get_table_names()
+    for table in tables:
+        if table.startswith("_alembic_tmp_"):
+            operations.drop_table(table)
     operations.execute("PRAGMA foreign_keys = OFF")
 
 
@@ -143,7 +129,7 @@ def add_pre_upgrade_hooks(autogen_context, upgrade_ops, schemas):
         upgrade_ops.ops.append(PostUpgradeOp())
 
 
-def alembic_config(connection, alembic_path):
+def alembic_config(connection: Engine, alembic_path: str):
     alembic_cfg = Config(os.path.join(alembic_path, "alembic.ini"),
                          attributes={
                              "configure_logging": False,
@@ -153,16 +139,15 @@ def alembic_config(connection, alembic_path):
     return alembic_cfg
 
 
-def migrate_database(connection, alembic_path=None):
+def migrate_database(connection: Engine, alembic_path: Optional[str] = None):
     if alembic_path is None:
         alembic_path = os.path.join(os.getcwd(), "database")
     assert os.path.exists(os.path.abspath(alembic_path)), f"alembic path {os.path.abspath(alembic_path)} does not exist"
     config = alembic_config(connection, alembic_path)
     command.upgrade(config, "head")
-    assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1, "foreign keys are not enabled"
 
-
-active_databases = {}
+    check = connection.execute("PRAGMA foreign_keys").fetchone()
+    assert check is not None and check[0] == 1, "foreign keys are not enabled"
 
 
 def register_bases(bases, show_deprecation_warning=True):
@@ -193,92 +178,7 @@ def run_env_py(bases):
     del sys.modules["openmodule.database.env"]  # unload the module, so we can re-run it (mostly testcases)
 
 
-def database_path(db_folder, db_name):
-    return os.path.join(db_folder, db_name) + ".sqlite3"
-
-
-def get_database(db_folder: str, name: str, alembic_path=None):
-    global active_databases
-    tmp = database_path(db_folder, name)
-    assert active_databases.get(tmp) is None, f"database {tmp} already exists," \
-                                              f" check if it was shutdown before a new one was created"
-    os.makedirs(db_folder, exist_ok=True)
-    path = f"sqlite:///{tmp}"
-    engine = create_engine(path, poolclass=StaticPool, connect_args={'check_same_thread': False})
-    migrate_database(engine, alembic_path)
-    active_databases[tmp] = engine
-
-    return engine
-
-
-class DatabaseContext:
-    def __init__(self, database: 'Database', expire_on_commit=True):
-        self.database = database
-        self.expire_on_commit = expire_on_commit
-
-    def __enter__(self) -> Session:
-        return self.database.__enter__(expire_on_commit=self.expire_on_commit)
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        return self.database.__exit__(exc_type, exc_val, exc_tb)
-
-
-class SessionWrapper(Session):
-    _closed = False
-
-    def __getattribute__(self, item):
-        attr = super().__getattribute__(item)
-        if isinstance(attr, types.MethodType) and self._closed:
-            raise AssertionError("Session is already closed")
-        return attr
-
-    def close(self):
-        super().close()
-        self._closed = True
-
-
-class Database:
-    active_session: Optional[Session]
-
-    def __init__(self, database_folder, name="database", alembic_path=None):
-        self.db_folder = database_folder
-        self.name = name
-        self._engine = get_database(database_folder, name, alembic_path)
-        self._session = sessionmaker(bind=self._engine, class_=SessionWrapper)
-        self.active_session = None
-        self.lock = threading.RLock()
-
-    def is_open(self):
-        return bool(self._session)
-
-    def shutdown(self):
-        assert self.is_open(), "database is already closed, you called shutdown twice somewhere"
-
-        with self.lock:
-            self._session = None
-            global active_databases
-            active_databases.pop(database_path(self.db_folder, self.name), None)
-
-    def __call__(self, expire_on_commit=True) -> DatabaseContext:
-        return DatabaseContext(self, expire_on_commit=expire_on_commit)
-
-    def __enter__(self, expire_on_commit=True) -> Session:
-        assert self._session, "Session is already closed"
-        self.lock.acquire()
-        self.active_session = self._session(expire_on_commit=expire_on_commit)
-        return self.active_session
-
-    def flush(self, objects=None):
-        assert self._session and self.active_session, "Session is already closed"
-        self.active_session.flush(objects)
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        try:
-            if exc_type is None:
-                self.active_session.commit()
-            else:
-                self.active_session.rollback()
-        finally:
-            self.active_session.close()
-            self.active_session = None
-            self.lock.release()
+def check_invalid_database_column_type(typ):
+    from openmodule import config
+    if config.run_checks():
+        assert not isinstance(typ, DateTime), "Do NOT use DateTime fields, use TZDateTime fields instead"

{openmodule-13.4.0 → openmodule-13.6.0}/openmodule/dispatcher.py

@@ -108,6 +108,8 @@ class MessageDispatcher:
         self.raise_validation_errors = raise_validation_errors
         self.raise_handler_errors = raise_handler_errors
         self.executor = executor or DummyExecutor()
+        self._shutdown = False
+        self._shutdown_lock = threading.Lock()
 
     @property
     def is_multi_threaded(self):
@@ -125,7 +127,9 @@ class MessageDispatcher:
         return True
 
     def shutdown(self, wait=True):
-        self.executor.shutdown(wait=wait)
+        with self._shutdown_lock:
+            self._shutdown = True
+            self.executor.shutdown(wait=wait)
 
     def unregister_handler(self, listener: Listener):
         for topic, listeners in self.listeners.items():
@@ -180,15 +184,22 @@ class MessageDispatcher:
         return listener
 
     def dispatch(self, topic: str, message: Union[Dict, BaseModel]):
-        assert isinstance(topic, str), "topic must be a string"
-
-        if isinstance(message, BaseModel):
-            message = message.dict()
-
-        listeners = self.listeners.get(topic, [])
-        for listener in listeners:
-            if listener.matches(message):
-                self.executor.submit(self.execute, listener, message)
+        with self._shutdown_lock:
+            if self._shutdown:
+                # We need to drop messages after shutdown somewhere.
+                # Cannot be done in the executor because it's a python builtin base class.
+                # Could also be done in core, but it's better here because messages and _messages_internal are
+                # both dispatched here.
+                return
+            assert isinstance(topic, str), "topic must be a string"
+
+            if isinstance(message, BaseModel):
+                message = message.dict()
+
+            listeners = self.listeners.get(topic, [])
+            for listener in listeners:
+                if listener.matches(message):
+                    self.executor.submit(self.execute, listener, message)
 
     def execute(self, listener: Listener, message: Dict):
         try:

{openmodule-13.4.0 → openmodule-13.6.0}/openmodule/logging.py

@@ -1,11 +1,12 @@
 import logging
+import sys
 
 
 def init_logging(core):
     assert hasattr(core.config, "LOG_LEVEL"), (
-        "LOG_LEVLE setting not found in your config. In order to use logging please add \n"
+        "LOG_LEVEL setting not found in your config. In order to use logging please add \n"
         "> LOG_LEVEL = config.log_level()\n"
         "to your config.py"
     )
-    logging.basicConfig(level=core.config.LOG_LEVEL)
+    logging.basicConfig(level=core.config.LOG_LEVEL, stream=sys.stdout)
     logging.captureWarnings(True)

{openmodule-13.4.0 → openmodule-13.6.0}/openmodule/messaging.py

@@ -12,18 +12,29 @@ from openmodule.dispatcher import MessageDispatcher
 from openmodule.models.base import ZMQMessage
 
 
+def _internal_get_pub_socket(context, config, linger=100) -> zmq.Socket:
+    socket: zmq.Socket = context.socket(zmq.PUB)
+    socket.setsockopt(zmq.LINGER, linger)
+    socket.setsockopt(zmq.TCP_KEEPALIVE, 1)
+    socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 3600)
+    socket.connect(config.BROKER_SUB)
+    return socket
+
+
 def get_sub_socket(context, config, linger=0) -> zmq.Socket:
-    socket = context.socket(zmq.SUB)
+    socket: zmq.Socket = context.socket(zmq.SUB)
     socket.setsockopt(zmq.LINGER, linger)
+    socket.setsockopt(zmq.TCP_KEEPALIVE, 1)
+    socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 3600)
     socket.connect(config.BROKER_PUB)
     return socket
 
 
 def get_pub_socket(context, config, linger=100) -> zmq.Socket:
-    socket = context.socket(zmq.PUB)
-    socket.setsockopt(zmq.LINGER, linger)
-    socket.connect(config.BROKER_SUB)
-    return socket
+    raise AssertionError(
+        "This function must not be used. Use core().publish instead (OM-697, OM-700). This is because the pub_socket "
+        "receives all subscriptions which will result in a memory leak until a message is sent to the socket."
+    )
 
 
 def wait_for_connection(dispatcher: MessageDispatcher, pub_socket=None, pub_lock=None, timeout=100):

{openmodule-13.4.0 → openmodule-13.6.0}/openmodule/models/base.py

@@ -22,12 +22,27 @@ def _donotuse(v, *, default):
 
 
 if config.run_checks():
-    from openmodule import checks
-
-    meta_kwargs = {"metaclass": checks.CheckingOpenModuleModel}
+    class CheckingOpenModuleModel(type(BaseModel), type):
+        def __new__(mcs, name, bases, dct):
+            cls = super().__new__(mcs, name, bases, dct)
+
+            # noinspection PyUnresolvedReferences
+            for field_name, field in cls.__fields__.items():
+                if field.type_ == datetime:
+                    assert "_timezone_validator" in field.class_validators, (
+                        "datetime fields must use the timezone_validator to ensure all datetime values "
+                        "are always naive. Otherwise runtime errors may occur depending on the isoformat used.\n"
+                        f"In class {name}, please add:\n"
+                        f'  _tz_{field_name} = timezone_validator("{field_name}")'
+                    )
+
+            return cls
+
+    meta_kwargs = {"metaclass": CheckingOpenModuleModel}
 else:
     meta_kwargs = {}
 
+
 ESCAPE_ASCII = re.compile(r'([^ -~])')
 
 
@@ -122,15 +137,9 @@ def datetime_to_timestamp(dt: datetime):
 
 class ZMQMessage(OpenModuleModel):
     timestamp: datetime = Field(default_factory=lambda: datetime.utcnow())
-    name: str
+    name: str = Field(default_factory=lambda: settings.NAME)
     type: str
 
-    def __init__(self, **kwargs):
-        name = kwargs.pop("name", None)
-        if name is None:
-            name = settings.NAME
-        super().__init__(name=name, **kwargs)
-
     _tz_timestamp = timezone_validator("timestamp")
 
     def publish_on_topic(self, pub_socket: zmq.Socket, topic: str):