openmodule 18.0.1__tar.gz → 18.1.0__tar.gz

{openmodule-18.0.1 → openmodule-18.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openmodule
-Version: 18.0.1
+Version: 18.1.0
 Summary: Libraries for developing the arivo openmodule
 Home-page: https://gitlab.com/arivo-public/device-python/openmodule.git
 Author: ARIVO
@@ -20,7 +20,7 @@ Requires-Dist: pyzmq~=26.2
 Requires-Dist: pyyaml<7,>=5.0
 Requires-Dist: editdistance~=0.8.1
 Requires-Dist: sqlalchemy~=2.0.0
-Requires-Dist: alembic<2,>=1.5.4
+Requires-Dist: alembic~=1.18.4
 Requires-Dist: requests<3,>=2.22
 Requires-Dist: python-dateutil~=2.9
 Requires-Dist: python-dotenv~=1.2.0

{openmodule-18.0.1 → openmodule-18.1.0}/openmodule/database/database.py

@@ -1,6 +1,8 @@
 import os
 import threading
+import traceback
 import types
+from multiprocessing import Process, Pipe, ProcessError
 from typing import Optional, Tuple
 
 from sqlalchemy import create_engine, event
@@ -8,11 +10,8 @@ from sqlalchemy.engine import Engine
 from sqlalchemy.orm import sessionmaker, Session
 from sqlalchemy.pool import StaticPool
 
-from multiprocessing import Process, Pipe, ProcessError
-import traceback
-
 
-class MigrationError(BaseException):
+class MigrationError(Exception):
     # Exception wrapper to create exception with traceback from child process
     pass
 

{openmodule-18.0.1 → openmodule-18.1.0}/openmodule/database/migration.py

@@ -1,14 +1,16 @@
 import os
 import shutil
 import sys
+import time
 import warnings
-from typing import Optional
+from glob import glob
 
 from alembic import command, context
 from alembic.autogenerate import comparators, renderers
 from alembic.config import Config
 from alembic.operations import Operations, MigrateOperation
 from alembic.runtime.migration import MigrationContext
+from alembic.util.langhelpers import DispatchPriority
 from sqlalchemy import MetaData, DateTime, inspect, text
 from sqlalchemy.engine import Engine
 
@@ -114,26 +116,32 @@ def post_downgrade(operations, operation):
 
 
 @renderers.dispatch_for(PreUpgradeOp)
-def render_create_sequence(autogen_context, op):
+def render_pre_upgrade(autogen_context, op):
     return "op.pre_upgrade()"
 
 
 @renderers.dispatch_for(PreDowngradeOp)
-def render_drop_sequence(autogen_context, op):
+def render_pre_downgrade(autogen_context, op):
     return "op.pre_downgrade()"
 
 
 @renderers.dispatch_for(PostUpgradeOp)
-def render_create_sequence(autogen_context, op):
+def render_post_upgrade(autogen_context, op):
     return "op.post_upgrade()"
 
 
 @renderers.dispatch_for(PostDowngradeOp)
-def render_drop_sequence(autogen_context, op):
+def render_post_downgrade(autogen_context, op):
     return "op.post_downgrade()"
 
 
-@comparators.dispatch_for("schema")
+# Copilot found this issue:
+# The default priority is MEDIUM, which means our `add_pre_upgrade_hooks`
+# has the same priority as the built-in comparators.
+# As a result, this routine was executed before the built-in comparators,
+# at a time when `upgrade_ops.ops` was still empty.
+# Therefore, no custom operations were available and no render function was called.
+@comparators.dispatch_for("schema", priority=DispatchPriority.LAST)
 def add_pre_upgrade_hooks(autogen_context, upgrade_ops, schemas):
     # only add those if any operations exist, otherwise we always have changes
     if len(upgrade_ops.ops):
@@ -151,10 +159,43 @@ def alembic_config(connection: Engine, alembic_path: str):
     return alembic_cfg
 
 
-def migrate_database(engine: Engine, alembic_path: Optional[str] = None):
+def validate_migration_files(alembic_path: str) -> bool:
+    """
+    This validation function is the final defense line against the problem from CORE-1593.
+    We do not apply any migration without our custom auto generated operations.
+    Because we need those operations to prevent data loss because of the foreign key behavior of SQLite.
+
+    This function raises a `ValueError` if a migration file is incorrect.
+    """
+    migration_folder = os.path.join(alembic_path, "alembic/versions/")
+    assert os.path.exists(os.path.abspath(migration_folder)), \
+        f"migration folder {os.path.abspath(migration_folder)} does not exist"
+    py_files = glob(os.path.join(migration_folder, "*.py"))
+    assert len(py_files) > 0, f"no python files found in {migration_folder}"
+    for py_file in glob(os.path.join(migration_folder, "*.py")):
+        if os.path.basename(py_file) == "__init__.py":
+            continue
+        with open(py_file) as f:
+            file_content = f.read()
+        if not all(op in file_content for op in ["op.pre_upgrade()", "op.pre_downgrade()",
+                                                 "op.pre_downgrade()", "op.post_downgrade()"]):
+            # We introduce a short sleep here because all services enter a crashing state,
+            # when the migrations are broken.
+            # With Docker's "restart: always" policy, it continuously tries to restart
+            # failing processes as fast as possible, which can lead to 100% CPU usage.
+            time.sleep(3)
+            raise ValueError("Not applying broken migration files! Our custom migration operations "
+                             f"are not in the migration file {os.path.basename(py_file)}. "
+                             "Please regenerate them with the correct alembic version from the Openmodule. "
+                             "For more information see CORE-1593.")
+    return True
+
+
+def migrate_database(engine: Engine, alembic_path: str | None = None):
     if alembic_path is None:
         alembic_path = os.path.join(os.getcwd(), "database")
     assert os.path.exists(os.path.abspath(alembic_path)), f"alembic path {os.path.abspath(alembic_path)} does not exist"
+    validate_migration_files(alembic_path)
     config = alembic_config(engine, alembic_path)
     command.upgrade(config, "head")
 

{openmodule-18.0.1 → openmodule-18.1.0}/openmodule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openmodule
-Version: 18.0.1
+Version: 18.1.0
 Summary: Libraries for developing the arivo openmodule
 Home-page: https://gitlab.com/arivo-public/device-python/openmodule.git
 Author: ARIVO
@@ -20,7 +20,7 @@ Requires-Dist: pyzmq~=26.2
 Requires-Dist: pyyaml<7,>=5.0
 Requires-Dist: editdistance~=0.8.1
 Requires-Dist: sqlalchemy~=2.0.0
-Requires-Dist: alembic<2,>=1.5.4
+Requires-Dist: alembic~=1.18.4
 Requires-Dist: requests<3,>=2.22
 Requires-Dist: python-dateutil~=2.9
 Requires-Dist: python-dotenv~=1.2.0

{openmodule-18.0.1 → openmodule-18.1.0}/openmodule.egg-info/requires.txt

@@ -5,7 +5,7 @@ pyzmq~=26.2
 pyyaml<7,>=5.0
 editdistance~=0.8.1
 sqlalchemy~=2.0.0
-alembic<2,>=1.5.4
+alembic~=1.18.4
 requests<3,>=2.22
 python-dateutil~=2.9
 python-dotenv~=1.2.0

{openmodule-18.0.1 → openmodule-18.1.0}/tests/test_database.py

@@ -1,7 +1,10 @@
 import os
+import shutil
 import time
+from glob import glob
+from tempfile import TemporaryDirectory
 from threading import Thread
-from unittest import TestCase
+from unittest import TestCase, mock
 from unittest.mock import patch
 
 import freezegun
@@ -13,7 +16,7 @@ from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound, DetachedInst
 
 from openmodule.config import settings
 from openmodule.database.database import Database, active_databases, database_path, MigrationError
-from openmodule.database.migration import alembic_config
+from openmodule.database.migration import alembic_config, validate_migration_files
 from openmodule.utils.db_helper import update_query, delete_query
 from openmodule_test.database import SQLiteTestMixin
 from tests.database_models_migration import DatabaseCascadeDeleteParentModel, DatabaseCascadeDeleteChildModel
@@ -458,3 +461,35 @@ class DatabaseMigrationProcessFailTest(TestCase):
         with patch("openmodule.database.database.MigrationProcess.join", side_effect=TimeoutError), \
                 self.assertRaises(TimeoutError):
             _ = Database(self.get_database_folder(), alembic_path=alembic_path)
+
+
+class MigrationFileValidationTestCase(TestCase):
+    def test_alembic_version_folder_does_not_exist(self):
+        with self.assertRaises(AssertionError) as cm:
+            validate_migration_files("/does/not/exist")
+        self.assertEqual("migration folder /does/not/exist/alembic/versions does not exist", str(cm.exception))
+
+    def test_no_python_file_are_available(self):
+        td = TemporaryDirectory(prefix="__openmodule_test_case__")
+        shutil.copytree("../tests/migration_file_broken", td.name, dirs_exist_ok=True)
+        migration_folder = os.path.join(td.name, "alembic/versions/")
+        py_files = glob(os.path.join(migration_folder, "*.py"))
+        for py_file in py_files:
+            os.unlink(py_file)
+        with self.assertRaises(AssertionError) as cm:
+            validate_migration_files(td.name)
+        self.assertIn("no python files found in /tmp/__openmodule_test_case__", str(cm.exception))
+
+    @mock.patch("openmodule.database.migration.time.sleep")
+    def test_validation_throw_value_error_for_broken_migration_file(self, time_sleep_mock: mock.MagicMock):
+        with self.assertRaises(ValueError) as cm:
+            validate_migration_files("../tests/migration_file_broken")
+        time_sleep_mock.assert_called_once()
+        self.assertEqual("Not applying broken migration files! "
+                         "Our custom migration operations are not in the migration file 46b275099a0d_initial.py. "
+                         "Please regenerate them with the correct alembic version from the Openmodule. "
+                         "For more information see CORE-1593.", str(cm.exception))
+
+    def test_validation_does_not_throw_value_error_for_correct_migration_file(self):
+        result = validate_migration_files("../tests/migration_test_database")
+        self.assertTrue(result)