django-overlay 0.1.0__py3-none-any.whl

django_overlay/_templating.py

@@ -0,0 +1,22 @@
+"""Shared Jinja env for sql_templates/. Split out of sql.py so
+strategies.py can render templates too without a circular import."""
+
+import jinja2
+
+
+def _qi(name: str) -> str:
+    """Jinja filter: quote a Postgres identifier, e.g. reserved words like `order`."""
+    return '"' + name.replace('"', '""') + '"'
+
+
+_ENV = jinja2.Environment(
+    loader=jinja2.PackageLoader("django_overlay", "sql_templates"),
+    autoescape=False,
+    trim_blocks=True,
+    lstrip_blocks=True,
+)
+_ENV.filters["qi"] = _qi
+
+
+def render(template_name: str, **context) -> str:
+    return _ENV.get_template(template_name).render(**context).strip()

django_overlay/apps.py

@@ -0,0 +1,20 @@
+from django.apps import AppConfig
+from django.core.exceptions import ImproperlyConfigured
+
+
+class DjangoOverlayConfig(AppConfig):
+    name = "django_overlay"
+    verbose_name = "Django Overlay"
+    default_auto_field = "django.db.models.AutoField"
+
+    def ready(self):
+        from . import checks  # also registers the manage.py check
+
+        # Runs on every process boot, not just `manage.py check` — a bad
+        # FK/M2M against an overlay model should fail loudly right away.
+        errors = checks.check_no_plain_fk_to_overlay_models(None)
+        if errors:
+            raise ImproperlyConfigured(
+                "django_overlay found unsafe references to overlay view models:\n"
+                + "\n".join(f"{error.id}: {error.msg}" for error in errors)
+            )

django_overlay/checks.py

@@ -0,0 +1,59 @@
+from django.apps import apps
+from django.core import checks
+from django.db import models
+
+from .fields import OverlayForeignKey
+
+
+@checks.register(checks.Tags.models)
+def check_no_plain_fk_to_overlay_models(app_configs, **kwargs):
+    """Fails `manage.py check` for a plain ForeignKey/OneToOneField
+    pointing at an OverlayModel, or a ManyToManyField pointing at one
+    without an explicit through= model."""
+    errors = []
+    for model in apps.get_models(include_auto_created=True):
+        for field in model._meta.get_fields():
+            if not field.is_relation:
+                continue
+            related_model = field.related_model
+            if related_model is None or not getattr(related_model, "_is_overlay_view_model", False):
+                continue
+
+            # M2M fields are never "concrete" (even the declaring side), so
+            # check the type directly instead.
+            if isinstance(field, models.ManyToManyField):
+                if not field.remote_field.through._meta.auto_created:
+                    continue
+                errors.append(
+                    checks.Error(
+                        f"{model.__name__}.{field.name} is a plain ManyToManyField pointing at "
+                        f"{related_model.__name__}, which is a django_overlay view model. Django's "
+                        "auto-created through table would use a plain ForeignKey, which can't hold "
+                        "a real FK constraint against a view.",
+                        hint="Use django_overlay.fields.OverlayManyToManyField (through=... is "
+                        "required) or write your own explicit through model with OverlayForeignKey "
+                        "fields.",
+                        obj=field,
+                        id="django_overlay.E002",
+                    )
+                )
+                continue
+
+            # Every concrete, non-M2M relation field Django has is a
+            # ForeignKey or OneToOneField (or a subclass), so this is
+            # already effectively "is a plain FK/O2O".
+            if not getattr(field, "concrete", False):
+                continue
+            if isinstance(field, OverlayForeignKey):
+                continue
+            errors.append(
+                checks.Error(
+                    f"{model.__name__}.{field.name} is a plain {type(field).__name__} pointing at "
+                    f"{related_model.__name__}, which is a django_overlay view model. Postgres cannot "
+                    f"hold a real FK constraint against a view.",
+                    hint=f"Use django_overlay.fields.OverlayForeignKey instead of {type(field).__name__}.",
+                    obj=field,
+                    id="django_overlay.E001",
+                )
+            )
+    return errors

django_overlay/constraints.py

@@ -0,0 +1,18 @@
+from django.db import models
+
+from .models import OverlayConfigurationError
+
+
+class OverlayUniqueConstraint(models.UniqueConstraint):
+    """Also guards against a value already present in the source table.
+    Postgres can't enforce that across a view's UNION ALL, so that half is
+    a trigger instead (see AddOverlayUniqueConstraint)."""
+
+    def __init__(self, *args, **kwargs):
+        if kwargs.get("condition") is not None:
+            raise OverlayConfigurationError(
+                "OverlayUniqueConstraint doesn't support condition= — the source-vs-base trigger "
+                "has no way to apply it, so it would silently check for collisions the condition "
+                "should have excluded."
+            )
+        super().__init__(*args, **kwargs)

django_overlay/fields.py

@@ -0,0 +1,76 @@
+from django.db import models
+
+from .models import OverlayConfigurationError
+from .strategies import negates_source_ids
+
+
+class OverlayForeignKey(models.ForeignKey):
+    """FK for pointing at an OverlayModel. Postgres can't hold a real FK
+    against a view, so this never creates a db constraint; referential
+    integrity instead comes from a constraint trigger (see
+    operations.AddOverlayConstraint)."""
+
+    def __init__(self, to, *args, **kwargs):
+        if "db_constraint" in kwargs:
+            raise OverlayConfigurationError(
+                "OverlayForeignKey always sets db_constraint=False (Postgres can't hold a real FK "
+                "against a view) — don't pass db_constraint yourself."
+            )
+        kwargs["db_constraint"] = False
+        super().__init__(to, *args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        kwargs.pop("db_constraint", None)
+        return name, path, args, kwargs
+
+    def trigger_name(self, model) -> str:
+        return f"overlayfk_{model._meta.db_table}_{self.column}"[:63]
+
+    def target_tables(self, tenant_schema: str) -> list[dict]:
+        """The target's base table plus its source, if any — see target_tables_for()."""
+        return target_tables_for(self.remote_field.model, tenant_schema)
+
+
+def target_tables_for(target, tenant_schema: str) -> list[dict]:
+    """[{"schema", "table", "id_column", "negate", "soft_delete"}, ...] for
+    `target`'s base table (never negated) plus its source, if any (negated
+    for a NEGATIVE_ID target). Takes the target model directly, not `self`, so a
+    migration operation can call it against a *live* model even when the
+    referencing field only exists in migration-historical state."""
+    tables = [
+        {
+            "schema": tenant_schema,
+            "table": target._base_model._meta.db_table,
+            "id_column": "id",
+            "negate": False,
+            "soft_delete": target._overlay_meta.soft_delete,
+        }
+    ]
+    source = target.get_source()
+    if source is not None:
+        negate = negates_source_ids(target._overlay_meta.strategy)
+        tables.append(
+            {
+                "schema": source.schema,
+                "table": source.table,
+                "id_column": source.id_column,
+                "negate": negate,
+                "soft_delete": False,
+            }
+        )
+    return tables
+
+
+class OverlayOneToOneField(OverlayForeignKey, models.OneToOneField):
+    pass
+
+
+class OverlayManyToManyField(models.ManyToManyField):
+    """M2M field for relating to an OverlayModel. Requires an explicit
+    through= model with OverlayForeignKey fields — Django's auto-created
+    through table always uses a plain ForeignKey, which can never be safe
+    against a view."""
+
+    def __init__(self, to, *args, through, **kwargs):
+        super().__init__(to, *args, through=through, **kwargs)

django_overlay/management/commands/makemigrations.py

@@ -0,0 +1,156 @@
+from django.apps import apps as django_apps
+from django.core.management.commands.makemigrations import Command as BaseCommand
+from django.db import migrations
+
+from ...constraints import OverlayUniqueConstraint
+from ...fields import OverlayForeignKey
+from ...models import OverlayModelBase
+from ...operations import (
+    AddOverlayConstraint,
+    AddOverlayUniqueConstraint,
+    DropOverlayView,
+    RemoveOverlayConstraint,
+    RemoveOverlayUniqueConstraint,
+    SyncOverlayView,
+)
+
+
+# Operation types whose references_model/references_field are actually
+# implemented (the Operation base class default just returns True always).
+# IndexOperation subclasses (AddConstraint, RemoveConstraint) fall in that
+# "always True" bucket, so they're handled separately below.
+_FIELD_SHAPE_OPS = (
+    migrations.AddField,
+    migrations.RemoveField,
+    migrations.AlterField,
+    migrations.RenameField,
+    migrations.CreateModel,
+    migrations.RenameModel,
+)
+_FK_RETRIGGER_OPS = (migrations.RenameField, migrations.AlterField)
+
+
+def _overlay_base_to_view(app_label):
+    """{base_model_name (lowercase): view_model_name} for every OverlayModel
+    in this app."""
+    return {
+        model._meta.model_name: model._view_model.__name__
+        for model in django_apps.get_app_config(app_label).get_models()
+        if isinstance(model, OverlayModelBase) and hasattr(model, "_view_model")
+    }
+
+
+def _overlay_foreign_key_fields(app_label):
+    """{(model_name (lowercase), field_name)} for every OverlayForeignKey
+    field declared anywhere in this app — the declaring model doesn't have
+    to be an OverlayModel itself (e.g. a bonus FK/through table)."""
+    fields = set()
+    for model in django_apps.get_app_config(app_label).get_models(include_auto_created=True):
+        for field in model._meta.get_fields():
+            if isinstance(field, OverlayForeignKey) and getattr(field, "concrete", False):
+                fields.add((model._meta.model_name, field.name))
+    return fields
+
+
+def _new_overlay_unique_constraints(op):
+    """[(model_name, constraint), ...] for OverlayUniqueConstraints this
+    operation introduces — from a brand-new model's `options["constraints"]`
+    (post-optimizer shape) or a standalone AddConstraint on an existing one."""
+    if isinstance(op, migrations.CreateModel):
+        return [(op.name, c) for c in op.options.get("constraints", []) if isinstance(c, OverlayUniqueConstraint)]
+    if isinstance(op, migrations.AddConstraint) and isinstance(op.constraint, OverlayUniqueConstraint):
+        return [(op.model_name, op.constraint)]
+    return []
+
+
+def _insert_view_drops_before_deletes(app_label, operations):
+    """A DeleteModel drops the model's table, but its view (if any) depends
+    on it — Postgres refuses to DROP TABLE while a dependent view exists.
+    Insert a DropOverlayView immediately before every DeleteModel,
+    unconditionally, same reasoning as RemoveField/RemoveConstraint below:
+    DROP VIEW IF EXISTS makes it safe even for a plain model with no view."""
+    new_operations = []
+    for op in operations:
+        if isinstance(op, migrations.DeleteModel):
+            new_operations.append(DropOverlayView(app_label, op.name))
+        new_operations.append(op)
+    return new_operations
+
+
+def extra_ops_for_migration(app_label, operations, base_to_view, fk_fields):
+    """The extra operations one migration should get, given this app's
+    OverlayModel registry (see _overlay_base_to_view/_overlay_foreign_key_fields).
+    Split out of Command.write_migration_files so it's directly testable.
+
+    RemoveField/RemoveConstraint are handled unconditionally, not keyed
+    against the registry: the removed field/constraint is already gone from
+    live code by the time this runs, so there's nothing to check its type
+    against. DROP TRIGGER IF EXISTS makes appending them speculatively safe."""
+    extra_ops = []
+    synced_views = set()
+    added_constraints = set()
+    added_unique_constraints = set()
+
+    for op in operations:
+        if isinstance(op, _FIELD_SHAPE_OPS):
+            for base_name, view_name in base_to_view.items():
+                if view_name not in synced_views and op.references_model(base_name, app_label):
+                    synced_views.add(view_name)
+                    extra_ops.append(SyncOverlayView(app_label, view_name))
+
+        if isinstance(op, migrations.RemoveField):
+            extra_ops.append(RemoveOverlayConstraint(app_label, op.model_name, op.name))
+
+        if isinstance(op, migrations.RemoveConstraint):
+            view_name = base_to_view.get(op.model_name.lower())
+            if view_name is not None:
+                extra_ops.append(RemoveOverlayUniqueConstraint(app_label, view_name, op.name))
+
+        if isinstance(op, migrations.AddField) and isinstance(op.field, OverlayForeignKey):
+            key = (op.model_name.lower(), op.name)
+            if key not in added_constraints:
+                added_constraints.add(key)
+                extra_ops.append(AddOverlayConstraint(app_label, op.model_name, op.name))
+        elif isinstance(op, migrations.CreateModel):
+            for field_name, field in op.fields:
+                if isinstance(field, OverlayForeignKey):
+                    key = (op.name.lower(), field_name)
+                    if key not in added_constraints:
+                        added_constraints.add(key)
+                        extra_ops.append(AddOverlayConstraint(app_label, op.name, field_name))
+        elif isinstance(op, _FK_RETRIGGER_OPS):
+            for model_name, field_name in fk_fields:
+                key = (model_name, field_name)
+                if key not in added_constraints and op.references_field(model_name, field_name, app_label):
+                    added_constraints.add(key)
+                    extra_ops.append(AddOverlayConstraint(app_label, model_name, field_name))
+
+        for model_name, constraint in _new_overlay_unique_constraints(op):
+            view_name = base_to_view.get(model_name.lower())
+            if view_name is None:
+                continue
+            key = (model_name.lower(), constraint.name)
+            if key not in added_unique_constraints:
+                added_unique_constraints.add(key)
+                extra_ops.append(AddOverlayUniqueConstraint(app_label, view_name, constraint.name))
+
+    return extra_ops
+
+
+class Command(BaseCommand):
+    """Same as Django's makemigrations, except any operation touching an
+    OverlayModel's base table gets a SyncOverlayView appended, any
+    OverlayForeignKey/OverlayUniqueConstraint add/rename/remove gets its
+    matching Add/RemoveOverlay(Unique)Constraint appended, and any
+    DeleteModel gets a DropOverlayView inserted just before it."""
+
+    def write_migration_files(self, changes, *args, **kwargs):
+        for app_label, app_migrations in changes.items():
+            base_to_view = _overlay_base_to_view(app_label)
+            fk_fields = _overlay_foreign_key_fields(app_label)
+            for migration in app_migrations:
+                migration.operations = _insert_view_drops_before_deletes(app_label, migration.operations)
+                migration.operations.extend(
+                    extra_ops_for_migration(app_label, migration.operations, base_to_view, fk_fields)
+                )
+        return super().write_migration_files(changes, *args, **kwargs)

django_overlay/management/commands/resync_overlay_views.py

@@ -0,0 +1,30 @@
+from django.apps import apps as django_apps
+from django.core.management.base import BaseCommand, CommandError
+
+from ...sync import resync_view
+
+
+class Command(BaseCommand):
+    help = (
+        "Regenerate one or more OverlayModel views + triggers from their "
+        "current field list and get_source(), without a migration. Use "
+        "this whenever a tenant's resolved source changes without a field "
+        "change (e.g. moved from one vendor's table to another's)."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument("models", nargs="+", metavar="app_label.ModelName")
+        parser.add_argument("--database", default="default")
+
+    def handle(self, *args, **options):
+        for label in options["models"]:
+            try:
+                app_label, model_name = label.split(".")
+            except ValueError:
+                raise CommandError(f"Expected app_label.ModelName, got {label!r}") from None
+            try:
+                model = django_apps.get_model(app_label, model_name)
+            except LookupError as exc:
+                raise CommandError(str(exc)) from exc
+            resync_view(model, using=options["database"])
+            self.stdout.write(self.style.SUCCESS(f"Resynced {label}"))

django_overlay/models.py

@@ -0,0 +1,195 @@
+import copy
+
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.db import models
+
+from .strategies import Strategy, default_id_field
+
+
+class OverlayConfigurationError(Exception):
+    pass
+
+
+def _default_strategy() -> Strategy:
+    """Lets a project set its own default strategy via
+    settings.DJANGO_OVERLAY_DEFAULT_STRATEGY instead of Strategy.UUID4. A
+    model can still override this with .with_strategy(...)."""
+    configured = getattr(settings, "DJANGO_OVERLAY_DEFAULT_STRATEGY", Strategy.UUID4)
+    if not isinstance(configured, Strategy):
+        raise ImproperlyConfigured(
+            "settings.DJANGO_OVERLAY_DEFAULT_STRATEGY must be a django_overlay.strategies.Strategy "
+            f"member (e.g. Strategy.NEGATIVE_ID), got {configured!r}."
+        )
+    return configured
+
+
+def _default_soft_delete() -> bool:
+    """Lets a project opt every model into soft delete by default via
+    settings.DJANGO_OVERLAY_DEFAULT_SOFT_DELETE. A model can still override
+    this with its own OverlayMeta.soft_delete."""
+    configured = getattr(settings, "DJANGO_OVERLAY_DEFAULT_SOFT_DELETE", False)
+    if not isinstance(configured, bool):
+        raise ImproperlyConfigured(f"settings.DJANGO_OVERLAY_DEFAULT_SOFT_DELETE must be a bool, got {configured!r}.")
+    return configured
+
+
+# Options that emit DDL go on the base (managed=True) model; everything
+# else (ordering, verbose_name, ...) goes on the view model, since that's
+# what's actually queried.
+_BASE_ONLY_META_OPTIONS = ("constraints", "indexes", "unique_together", "index_together", "db_table_comment")
+
+# Neither model is a sound home for these: the view model is unmanaged, so
+# create_permissions() silently skips it; the base model isn't something
+# app code should reference. Reject instead of picking a bad default.
+_UNSUPPORTED_META_OPTIONS = ("permissions", "default_permissions")
+
+# Both models must agree on which app they belong to — Django can usually
+# infer this from the module either model is defined in, but an explicit
+# override needs to reach both, not just whichever side it happened to land on.
+_BOTH_META_OPTIONS = ("app_label",)
+
+# The metaclass sets these itself on both models — declaring your own would
+# just get silently overwritten, so reject instead.
+_FORCED_META_OPTIONS = {
+    "db_table": "table naming is controlled entirely by OverlayMeta.table_name (defaults to the lowercased class name)",
+    "managed": "the base model is always managed=True and the view model is always managed=False",
+}
+
+
+def _split_meta_options(model_name: str, user_meta) -> tuple[dict, dict]:
+    if user_meta is None:
+        return {}, {}
+    options = {k: v for k, v in vars(user_meta).items() if not k.startswith("_")}
+    forced = [k for k in _FORCED_META_OPTIONS if k in options]
+    if forced:
+        raise OverlayConfigurationError(
+            f"{model_name}.Meta.{forced[0]} isn't supported on an OverlayModel — "
+            f"{_FORCED_META_OPTIONS[forced[0]]}; it would just be silently overwritten."
+        )
+    unsupported = [k for k in _UNSUPPORTED_META_OPTIONS if k in options]
+    if unsupported:
+        raise OverlayConfigurationError(
+            f"{model_name}.Meta.{unsupported[0]} isn't supported on an OverlayModel — there's no "
+            "model to attach it to that makes sense (see _UNSUPPORTED_META_OPTIONS)."
+        )
+    base_options = {k: v for k, v in options.items() if k in _BASE_ONLY_META_OPTIONS + _BOTH_META_OPTIONS}
+    view_options = {k: v for k, v in options.items() if k not in _BASE_ONLY_META_OPTIONS}
+    return base_options, view_options
+
+
+class OverlayMeta:
+    """Base class for a model's inner OverlayMeta. Subclass it (directly,
+    or via with_strategy()) and add table_name / get_source()."""
+
+    Strategy = Strategy
+    strategy = _default_strategy()
+    soft_delete = _default_soft_delete()
+    pk_default_sql = None
+
+    @classmethod
+    def with_strategy(cls, strategy: Strategy):
+        return type(f"OverlayMeta_{strategy.value}", (cls,), {"strategy": strategy})
+
+    @staticmethod
+    def get_source():
+        raise NotImplementedError("OverlayMeta subclasses must implement get_source().")
+
+
+class OverlayModelBase(models.base.ModelBase):
+    """Splits one `class Person(OverlayModel)` into a hidden managed=True
+    base table and the managed=False view model the app actually imports."""
+
+    def __new__(mcs, name, bases, namespace, **kwargs):
+        if namespace.pop("_overlay_root", False):
+            return super().__new__(mcs, name, bases, namespace, **kwargs)
+
+        is_overlay_subclass = any(isinstance(b, OverlayModelBase) for b in bases)
+        meta = namespace.get("Meta")
+        if not is_overlay_subclass or (meta is not None and getattr(meta, "abstract", False)):
+            return super().__new__(mcs, name, bases, namespace, **kwargs)
+
+        overlay_meta = namespace.pop("OverlayMeta", None)
+        if overlay_meta is None or not issubclass(overlay_meta, OverlayMeta):
+            raise OverlayConfigurationError(f"{name}.OverlayMeta must subclass django_overlay.models.OverlayMeta.")
+        if "get_source" not in overlay_meta.__dict__:
+            raise OverlayConfigurationError(
+                f"{name}.OverlayMeta must implement get_source() returning a SourceTable | None."
+            )
+        if not isinstance(overlay_meta.strategy, Strategy):
+            raise OverlayConfigurationError(
+                f"{name}.OverlayMeta.strategy must be a Strategy member (e.g. Strategy.NEGATIVE_ID "
+                f"or .with_strategy(...)), got {overlay_meta.strategy!r}."
+            )
+        if not isinstance(overlay_meta.soft_delete, bool):
+            raise OverlayConfigurationError(
+                f"{name}.OverlayMeta.soft_delete must be a bool, got {overlay_meta.soft_delete!r}."
+            )
+
+        # M2M fields go on the view model only — copying one to both models
+        # would create two independent through tables.
+        m2m_items = {k: v for k, v in namespace.items() if isinstance(v, models.ManyToManyField)}
+        field_items = {k: v for k, v in namespace.items() if isinstance(v, models.Field) and k not in m2m_items}
+        rest_items = {k: v for k, v in namespace.items() if k not in field_items and k not in m2m_items}
+        table_name = getattr(overlay_meta, "table_name", name.lower())
+
+        if overlay_meta.soft_delete and "_overlay_deleted" in field_items:
+            raise OverlayConfigurationError(
+                f"{name} can't declare its own `_overlay_deleted` field — django_overlay reserves "
+                "that name for its soft_delete shadow flag."
+            )
+
+        if "id" not in field_items:
+            injected = default_id_field(overlay_meta.strategy)
+            if injected is not None:
+                field_items["id"] = injected
+
+        base_meta_options, view_meta_options = _split_meta_options(name, namespace.get("Meta"))
+
+        base_ns = {**rest_items, **{k: copy.deepcopy(v) for k, v in field_items.items()}}
+        if overlay_meta.soft_delete:
+            # Base-only shadow flag — never copied to the view model, so it
+            # never shows up as a queryable column there.
+            base_ns["_overlay_deleted"] = models.BooleanField(default=False, editable=False)
+        base_ns["__qualname__"] = f"{name}Base"
+        # No default_permissions for the base table — nobody should see
+        # "Can add <name>base" in an admin permission list.
+        base_ns["Meta"] = type("Meta", (), {**base_meta_options, "db_table": table_name, "default_permissions": ()})
+        base_model = super().__new__(mcs, f"{name}Base", bases, base_ns, **kwargs)
+
+        view_ns = {**rest_items, **{k: copy.deepcopy(v) for k, v in field_items.items()}, **m2m_items}
+        view_ns["Meta"] = type("Meta", (), {**view_meta_options, "db_table": f"{table_name}_view", "managed": False})
+        view_model = super().__new__(mcs, name, bases, view_ns, **kwargs)
+
+        view_model._base_model = base_model
+        view_model._overlay_meta = overlay_meta
+        view_model._is_overlay_view_model = True
+        base_model._view_model = view_model
+        return view_model
+
+
+class OverlayModel(models.Model, metaclass=OverlayModelBase):
+    _overlay_root = True
+
+    Strategy = Strategy
+
+    class Meta:
+        abstract = True
+
+    @classmethod
+    def base_table(cls):
+        """The hidden concrete model backing this view. Migration/tooling
+        use only — application code should never query or write to it."""
+        return cls._base_model
+
+    @classmethod
+    def get_source(cls):
+        return cls._overlay_meta.get_source()
+
+    def reset_to_source(self):
+        """Discard this row's local materialization/soft-deletion and fall
+        back to whatever the source shows for its id (nothing, if there's no
+        source row). Not a delete — doesn't run Django's on_delete collector,
+        since the identity itself isn't necessarily going away. See
+        docs/concepts/DELETION.md."""
+        self._base_model.objects.filter(pk=self.pk).delete()