mainsequence-metatable 0.1.12.dev24__tar.gz → 0.1.12.dev25__tar.gz

{mainsequence_metatable-0.1.12.dev24 → mainsequence_metatable-0.1.12.dev25}/.agents/skills/client/metatables-local-development/SKILL.md

@@ -149,7 +149,8 @@ requires those checks. Application models keep their hosted column types: a
 PostgreSQL `JSONB` column runs as JSON, and `Numeric` runs as a SQLite number exact
 to about 15 significant digits, the precision the client's float64 frames carry on
 every engine. Unsigned 64-bit integers and arrays are rejected, naming the column.
-Revision code that runs PostgreSQL-only SQL must check `op.get_bind().dialect.name`. Do not use the shared environment database as the default
+Revisions need no SQLite variants: column and constraint changes run through Alembic
+batch mode. Only raw PostgreSQL SQL must check `op.get_bind().dialect.name`. Do not use the shared environment database as the default
 integration-test fixture.
 
 ## Finish verification and switch back

{mainsequence_metatable-0.1.12.dev24 → mainsequence_metatable-0.1.12.dev25}/.agents/skills/client/metatables-migrations/SKILL.md

@@ -154,6 +154,12 @@ The global `--local` selects the running project's API connection; inspect runti
 status because Admin can switch that API to Hosted. Upgrade/downgrade accept
 Alembic targets. Use `--no-autogenerate` for offline revision authoring.
 
+Write each revision once, for the hosted engine. On local SQLite, MetaTables runs
+column and constraint changes through Alembic batch mode, generated revisions use
+batch blocks, `JSONB` runs as JSON, and `postgresql_where` predicates also apply to
+SQLite. Raw PostgreSQL SQL, such as `ctid`, `::` casts or `jsonb_*` functions, must
+branch on `op.get_bind().dialect.name`.
+
 The client imports and executes application revisions, using the connection
 resolved for the selected environment. Environment operators supply the database
 login's DDL privileges. MetaTables does not mint migration roles or sandbox DDL.

{mainsequence_metatable-0.1.12.dev24/src/mainsequence_metatable.egg-info → mainsequence_metatable-0.1.12.dev25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence-metatable
-Version: 0.1.12.dev24
+Version: 0.1.12.dev25
 Summary: Main Sequence MetaTables Python client and standalone API
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License-Expression: MIT

{mainsequence_metatable-0.1.12.dev24 → mainsequence_metatable-0.1.12.dev25}/docs/adr/api/0001-unified-api-storage-and-local-sqlite.md

@@ -736,6 +736,8 @@ workflow as PostgreSQL. SQLite rejects named schemas, unsigned 64-bit integer,
 array, and specialized index contracts, naming the column or index. PostgreSQL
 JSONB is stored as JSON, and NUMERIC as SQLite numbers exact to about 15
 significant digits, the precision the client's float64 frames already carry.
+Revisions written for PostgreSQL run unchanged: column and constraint changes use
+Alembic batch mode, with foreign keys checked before commit.
 Timestamps require UTC offsets and preserve microseconds; finer precision is
 rejected rather than truncated. UUID coordinate filters use the same contract
 conversion for reads and deletes.

{mainsequence_metatable-0.1.12.dev24 → mainsequence_metatable-0.1.12.dev25}/docs/operations/local-runtime.md

@@ -149,8 +149,17 @@ created and decoded as JSON. `Numeric` columns are SQLite numbers, exact to abou
 significant digits; reads return `Decimal`, and the client's frames carry them as
 float64 on every engine. Unsigned 64-bit integers and arrays have no SQLite type:
 registration rejects them with `unsupported_sqlite_type`, naming the table and
-column. Revision code running PostgreSQL-only SQL, such as `ctid` or `jsonb_*`
-functions, must branch on `op.get_bind().dialect.name`.
+column.
+
+Application revisions written for PostgreSQL run unchanged. SQLite cannot alter
+columns or constraints in place, so the migration environment runs `alter_column`,
+`add_column`, `drop_column` and constraint operations through Alembic batch mode,
+which copies the table and swaps it in. Foreign keys are not enforced while the
+revisions run; `PRAGMA foreign_key_check` runs before commit and rolls back every
+revision if a key dangles. A `postgresql_where` predicate also becomes the SQLite
+partial-index predicate. Generated revisions use batch blocks, which PostgreSQL
+applies as plain `ALTER` statements. Raw PostgreSQL SQL, such as `ctid`, `::` casts
+or `jsonb_*` functions, must branch on `op.get_bind().dialect.name`.
 
 The SQLite adapter shares the request's transaction for catalog and physical work.
 Physical savepoints preserve rollback semantics and avoid two writers competing for

{mainsequence_metatable-0.1.12.dev24 → mainsequence_metatable-0.1.12.dev25}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mainsequence-metatable"
-version = "0.1.12.dev24"
+version = "0.1.12.dev25"
 description = "Main Sequence MetaTables Python client and standalone API"
 readme = "README.md"
 requires-python = ">=3.13"

{mainsequence_metatable-0.1.12.dev24 → mainsequence_metatable-0.1.12.dev25/src/mainsequence_metatable.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence-metatable
-Version: 0.1.12.dev24
+Version: 0.1.12.dev25
 Summary: Main Sequence MetaTables Python client and standalone API
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License-Expression: MIT

{mainsequence_metatable-0.1.12.dev24 → mainsequence_metatable-0.1.12.dev25}/src/mainsequence_metatable.egg-info/SOURCES.txt

@@ -392,6 +392,7 @@ src/metatables/migrations/provider.py
 src/metatables/migrations/registry.py
 src/metatables/migrations/runner.py
 src/metatables/migrations/scaffold.py
+src/metatables/migrations/sqlite.py
 src/metatables/migrations/tls.py
 src/metatables/migrations/templates/__init__.py
 src/metatables/migrations/templates/env.py.mako

{mainsequence_metatable-0.1.12.dev24 → mainsequence_metatable-0.1.12.dev25}/src/metatables/migrations/connection.py

@@ -3,21 +3,13 @@
 from contextlib import contextmanager
 
 from sqlalchemy import create_engine, event
-from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.engine import make_url
-from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.pool import NullPool
 
+from . import sqlite as _sqlite  # noqa: F401  Registers SQLite DDL and batch rules.
 from .tls import migration_tls_arguments
 
 
-@compiles(JSONB, "sqlite")
-def _sqlite_jsonb(_type, _compiler, **_kw):
-    # Applications declare PostgreSQL JSONB; the local runtime stores and
-    # decodes it as JSON, so models and reflected columns compare equal.
-    return "JSON"
-
-
 def version_schema(config, dialect):
     schema = config.attributes["mainsequence_migration_provider"].version_table_schema
     if dialect in {"sqlite", "mysql"} and schema in (None, "public"):
@@ -51,7 +43,9 @@ def migration_engine(config):
                 @event.listens_for(engine, "connect")
                 def sqlite_connect(connection, _record):
                     connection.isolation_level = None
-                    connection.execute("PRAGMA foreign_keys = ON")
+                    # Batch rebuilds drop and recreate referenced tables; the
+                    # environment runs PRAGMA foreign_key_check before commit.
+                    connection.execute("PRAGMA foreign_keys = OFF")
                     connection.execute("PRAGMA busy_timeout = 30000")
 
                 @event.listens_for(engine, "begin")

{mainsequence_metatable-0.1.12.dev24 → mainsequence_metatable-0.1.12.dev25}/src/metatables/migrations/env.py

@@ -15,6 +15,7 @@ def run_mainsequence_alembic_env(
 
     from metatables.migrations import apply_mainsequence_migration_role
     from metatables.migrations.connection import migration_engine
+    from metatables.migrations.sqlite import AUTO_BATCH_OPTION, check_foreign_keys
 
     def migration_provider() -> Any:
         provider = context.config.attributes.get("mainsequence_migration_provider")
@@ -78,8 +79,18 @@ def run_mainsequence_alembic_env(
             "include_object": include_object,
             "compare_type": True,
             "compare_server_default": True,
+            # Generated revisions alter tables in batch blocks: plain ALTERs on
+            # PostgreSQL, copy-and-swap on the local SQLite runtime.
+            "render_as_batch": True,
+            AUTO_BATCH_OPTION: connection is not None and connection.dialect.name == "sqlite",
         }
 
+    def run(connection) -> None:
+        with context.begin_transaction():
+            context.run_migrations()
+            if connection.dialect.name == "sqlite":
+                check_foreign_keys(connection)
+
     def run_migrations_offline() -> None:
         context.configure(
             url=context.config.get_main_option("sqlalchemy.url"),
@@ -95,16 +106,14 @@ def run_mainsequence_alembic_env(
         if connection is not None:
             apply_mainsequence_migration_role(connection, config)
             context.configure(connection=connection, **configure_kwargs(connection))
-            with context.begin_transaction():
-                context.run_migrations()
+            run(connection)
             return
 
         config.attributes.setdefault("mainsequence_migration_provider", migration_provider())
         with migration_engine(config) as engine, engine.connect() as connection:
             apply_mainsequence_migration_role(connection, config)
             context.configure(connection=connection, **configure_kwargs(connection))
-            with context.begin_transaction():
-                context.run_migrations()
+            run(connection)
 
     if context.is_offline_mode():
         run_migrations_offline()

mainsequence_metatable-0.1.12.dev25/src/metatables/migrations/sqlite.py

@@ -0,0 +1,82 @@
+"""Run application revisions written for PostgreSQL on the local SQLite runtime.
+
+SQLite cannot alter columns or constraints in place. Inside the MetaTables
+Alembic environment, those operations run through Alembic batch mode (copy the
+table, then swap it in), so revisions need no SQLite-specific code. PostgreSQL
+and other dialects keep Alembic's own implementations.
+"""
+
+from __future__ import annotations
+
+from alembic.operations import BatchOperations, Operations, ops
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.compiler import compiles
+
+AUTO_BATCH_OPTION = "metatables_sqlite_auto_batch"
+
+
+@compiles(JSONB, "sqlite")
+def _sqlite_jsonb(_type, _compiler, **_kw):
+    # Applications declare PostgreSQL JSONB; the local runtime stores and
+    # decodes it as JSON, so models and reflected columns compare equal.
+    return "JSON"
+
+
+def _table(operation):
+    if isinstance(operation, ops.CreateForeignKeyOp):
+        return operation.source_table, operation.kw.get("source_schema")
+    return operation.table_name, operation.schema
+
+
+def _batched(default):
+    def run(operations, operation):
+        if (
+            isinstance(operations, BatchOperations)
+            or not operations.migration_context.opts.get(AUTO_BATCH_OPTION)
+        ):
+            return default(operations, operation)
+        connection = operations.migration_context.connection
+        if connection.exec_driver_sql("PRAGMA foreign_keys").scalar():
+            # Rebuilding a parent table with enforcement on deletes or rejects
+            # its children; the environment checks every key after migrating.
+            raise RuntimeError("SQLite batch migrations require foreign_keys=OFF.")
+        name, schema = _table(operation)
+        with operations.batch_alter_table(name, schema=None if schema == "public" else schema) as batch:
+            batch.invoke(operation)
+
+    return run
+
+
+def _partial_index(default):
+    def run(operations, operation):
+        # Portable predicates such as "status = 'active'" keep the index partial.
+        if operations.migration_context.opts.get(AUTO_BATCH_OPTION):
+            kw = operation.kw
+            if "postgresql_where" in kw and "sqlite_where" not in kw:
+                kw["sqlite_where"] = kw["postgresql_where"]
+        return default(operations, operation)
+
+    return run
+
+
+for _operation in (
+    ops.AlterColumnOp,
+    ops.AddColumnOp,
+    ops.DropColumnOp,
+    ops.AddConstraintOp,
+    ops.DropConstraintOp,
+):
+    Operations._to_impl.dispatch_for(_operation, qualifier="sqlite")(
+        _batched(Operations._to_impl.dispatch(_operation))
+    )
+Operations._to_impl.dispatch_for(ops.CreateIndexOp, qualifier="sqlite")(
+    _partial_index(Operations._to_impl.dispatch(ops.CreateIndexOp))
+)
+
+
+def check_foreign_keys(connection) -> None:
+    """Fail the migration transaction when a revision left a dangling key."""
+    violations = connection.exec_driver_sql("PRAGMA foreign_key_check").fetchall()
+    if violations:
+        tables = sorted({str(row[0]) for row in violations})
+        raise RuntimeError(f"Migration left foreign-key violations in: {', '.join(tables)}.")

{mainsequence_metatable-0.1.12.dev24 → mainsequence_metatable-0.1.12.dev25}/tests/integrations/test_sqlite_operations.py

@@ -125,7 +125,7 @@ def test_sqlite_runs_postgresql_jsonb_as_json_and_numeric_as_numbers(sqlite_sour
     from sqlalchemy import Column, Integer, MetaData, Numeric, Table
     from sqlalchemy.dialects.postgresql import JSONB
 
-    import metatables.migrations.connection  # noqa: F401  Registers the SQLite JSONB rule.
+    import metatables.migrations.sqlite  # noqa: F401  Registers the SQLite JSONB rule.
     from metatables.api.backend.backends.adapters.sqlite_introspection import (
         introspect_sqlite_table,
     )
@@ -160,6 +160,52 @@ def test_sqlite_runs_postgresql_jsonb_as_json_and_numeric_as_numbers(sqlite_sour
     assert [row["id"] for row in selected] == [2, 1]
 
 
+def test_postgresql_style_alter_revisions_run_through_sqlite_batch_mode(tmp_path):
+    from alembic.migration import MigrationContext
+    from alembic.operations import Operations
+    from sqlalchemy import Column, Integer, String, event, text
+
+    from metatables.migrations.sqlite import AUTO_BATCH_OPTION, check_foreign_keys
+
+    engine = create_engine(f"sqlite:///{tmp_path / 'batch.sqlite'}")
+    event.listen(engine, "connect", lambda raw, _record: raw.execute("PRAGMA foreign_keys = OFF"))
+    with engine.begin() as connection:
+        connection.exec_driver_sql("CREATE TABLE parent (id INTEGER PRIMARY KEY, code TEXT)")
+        connection.exec_driver_sql("CREATE TABLE child (id INTEGER PRIMARY KEY, parent_id INTEGER, "
+                                   "CONSTRAINT fk_child_parent FOREIGN KEY (parent_id) REFERENCES parent (id) ON DELETE CASCADE)")
+        connection.exec_driver_sql("INSERT INTO parent VALUES (1, 'a'), (2, 'b')")
+        connection.exec_driver_sql("INSERT INTO child VALUES (10, 1), (11, 2)")
+        op = Operations(MigrationContext.configure(connection, opts={AUTO_BATCH_OPTION: True}))
+        # The same calls a PostgreSQL revision makes; none of them is native SQLite DDL.
+        op.add_column("parent", Column("status", String(), nullable=True))
+        op.alter_column("parent", "code", existing_type=String(), nullable=False)
+        op.create_unique_constraint("uq_parent_code", "parent", ["code"])
+        op.drop_constraint("fk_child_parent", "child", type_="foreignkey")
+        op.create_foreign_key("fk_child_parent_restrict", "child", "parent", ["parent_id"], ["id"])
+        op.alter_column("child", "parent_id", existing_type=Integer(), nullable=False)
+        op.create_index("ix_parent_active", "parent", ["status"], unique=True,
+                        postgresql_where=text("status = 'active'"))
+        check_foreign_keys(connection)
+        inspector = inspect(connection)
+        assert [fk["name"] for fk in inspector.get_foreign_keys("child")] == ["fk_child_parent_restrict"]
+        assert [c["nullable"] for c in inspector.get_columns("child") if c["name"] == "parent_id"] == [False]
+        assert [u["name"] for u in inspector.get_unique_constraints("parent")] == ["uq_parent_code"]
+        index = next(i for i in inspector.get_indexes("parent") if i["name"] == "ix_parent_active")
+        assert str(index["dialect_options"]["sqlite_where"]) == "status = 'active'"
+        # Rebuilding the parent kept every child row instead of cascading.
+        assert connection.exec_driver_sql("SELECT id, parent_id FROM child ORDER BY id").all() == [(10, 1), (11, 2)]
+        connection.exec_driver_sql("INSERT INTO child VALUES (12, 99)")
+        with pytest.raises(RuntimeError, match="child"):
+            check_foreign_keys(connection)
+        connection.exec_driver_sql("DELETE FROM child WHERE id = 12")
+    with engine.connect() as connection:
+        connection.exec_driver_sql("PRAGMA foreign_keys = ON")
+        with pytest.raises(RuntimeError, match="foreign_keys=OFF"):
+            Operations(MigrationContext.configure(connection, opts={AUTO_BATCH_OPTION: True})).add_column(
+                "parent", Column("note", String()))
+    engine.dispose()
+
+
 def test_sub_microsecond_timestamps_fail_before_serialization():
     from metatables.api.backend.contracts.values import utc_timestamp
     from metatables.dtype_codec import TIMESTAMP_TZ, serialize_remote_value