django-postpone-index 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

{django_postpone_index-0.0.2.dist-info → django_postpone_index-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: django-postpone-index
-Version: 0.0.2
+Version: 0.0.4
 Summary: Postpone index creation to provide Zero Downtime Migration feature
 Home-page: https://github.com/nnseva/django-postpone-index
 Author: Vsevolod Novikov
@@ -105,6 +105,7 @@ The following complex use cases are processed by the package.
 - Back Migration. The both, forward and backward migrations are processed.
 - Implicit index drop while removing the table. The Django doesn't issue a separate SQL to drop indexes of the dropped table.
 - Implicit index drop while removing the field. The Django doesn't issue a separate SQL to drop indexes related to the dropped column.
+- Rename field (column) name or model (table) name
 
 ## Using
 
@@ -169,6 +170,43 @@ calling the `apply_postponed` migration command again. All not-applied indexes w
 **NOTICE** the `apply_postponed` management command doesn't have any explicit locking mechanics. Avoid starting this
 command concurrently with itself or another `migrate` command on the same database.
 
+## Intermediate migration state
+
+Apart from standard Django migrations, using the `postpone_index` package leads to the *intermediate migration state*
+after the `migrate` management command finished:
+
+- new model structure is applied
+- indexes to be deleted are deleted
+- indexes to be created are *not created yet*
+
+You should be aware that if you introduce a new unique index or constraint, the database does not control uniqueness
+based on not yet created indexes at this time.
+
+Your code works now as expected everywhere, except the code which is based on new unique constraints introduced in applied migrations.
+
+Apply the `apply_postponed run` management command to make these new indexes work.
+
+Any error while `apply_postponed run` execution is stored in the `PostponedSQL` model instance.
+
+You can see erroneous lines using `apply_postponed list` command. See the `[E]` mark at the start of the line.
+
+You also can see the error details using the format parameter of the `apply_postponed list -f '... %(error)s'` management command.
+
+The `apply_postponed run -x` breaks execution on any error. You can see the error in the standard error or logging streams.
+
+The `apply_postponed run` (without `-x` parameter) doesn't stop on error, but outputs warning to the log stream instead.
+
+When the error happened, it most probably is caused by the non-unique records. Fix the data and try to execute
+`apply_postponed run` again to create an index.
+
+After the successfull `apply_postponed run` execution, the migration state is finalised to be equal as if you applied the migration
+without `postpone_index` package at all.
+
+The `apply_postponed run` management command marks all successfully executed `PostponedSQL` instances as `done`. You can see `[X]` mark
+at the start of the line produced by `apply_ponsponed list` management command.
+
+You can cleanup `done` instances using `apply_postponed cleanup` management command. This step is optional.
+
 ## Django testing
 
 Django migrates testing database before tests. Always use `POSTPONE_INDEX_IGNORE = True` settings to avoid postpone index

{django_postpone_index-0.0.2.dist-info → django_postpone_index-0.0.4.dist-info}/RECORD

@@ -1,23 +1,23 @@
 postpone_index/__init__.py,sha256=s1NgdtOn7SO7scs-uAaq1134F7Luh-JjB5Ghra3D2fQ,84
-postpone_index/_version.py,sha256=huLsL1iGeXWQKZ8bjwDdIWC7JOkj3wnzBh-HFMZl1PY,704
+postpone_index/_version.py,sha256=QlXZ5JTjE_pgpDaeHk0GTExkc75xUZFmd0hA7kGYCJ0,704
 postpone_index/admin.py,sha256=TAZxaciQkEBNhReYulsU5VopPCyABtMLZvCSxk0CfYE,820
 postpone_index/apps.py,sha256=1ap2HY49uXVXjY9UWK3___fjPe22wqFOSOnuH2kIAxs,200
 postpone_index/migration_utils.py,sha256=02rm77mz_MAufl5vdqKPV1CAd1-3bRk6kSy4MIxKsEo,752
-postpone_index/models.py,sha256=jMsA5hdiL8Xgl-hq7bgVl5NUX6LXlSYpMuX9pD68BbU,1771
+postpone_index/models.py,sha256=R_S8KBKlfPy6A6Urwdp17IINcd9VEZtkW5_cVbpKN-4,3581
 postpone_index/testing_utils.py,sha256=ycxp7ovY6JceOvNpotK7OGAebhdHicXJ-0fWJmaq9fo,1224
-postpone_index/utils.py,sha256=wYVPsQRs3LN-ZtvfSWrDDqFFp0OCkVxW9j7w9yqd47o,3823
+postpone_index/utils.py,sha256=H_2M7IvSs1BKxvjkFanyBmX5I2iJB2KDDOAcINKlD04,4758
 postpone_index/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 postpone_index/contrib/postgis/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 postpone_index/contrib/postgis/base.py,sha256=a5h1k11BVSOflSSesTxObKAEYNWtisFLRbApzHeW3c0,289
 postpone_index/contrib/postgis/schema.py,sha256=eantuNhFYWDngDnAeURJurpW58UztPnjTIySmUlXx7s,340
 postpone_index/contrib/postgres/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 postpone_index/contrib/postgres/base.py,sha256=Xq90nJkxGSgR8Im9JPnFepeMb0-bur9sRjPpm3zMd6U,281
-postpone_index/contrib/postgres/schema.py,sha256=45xt6mhVgGgFWYH_OVZE0WzcyBv9J6sCQVR4WnxA1Xc,10914
+postpone_index/contrib/postgres/schema.py,sha256=tbqNy8l-YfR-h_snNaiHe2J4Uxb2JH5pZSzNDrfxtUQ,12690
 postpone_index/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 postpone_index/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 postpone_index/management/commands/apply_postponed.py,sha256=wVbw0ugUyPuYwUikGnn5MvP9l9OsEIZja9sb6nsMtOw,9107
 postpone_index/sql/start.sql,sha256=44yFbZTjknjXUvl7MpO1uuPv-CVBsZcnUDuKc8mVj3k,871
-django_postpone_index-0.0.2.dist-info/METADATA,sha256=8jnBu6D07a_Nszgmkk9e9kP7wMJzeE3N9jCsxc5PGOk,10415
-django_postpone_index-0.0.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-django_postpone_index-0.0.2.dist-info/top_level.txt,sha256=B6q_TICqApvruf_NJfnLFNLZLpxMLebQF6cPdKrWm5A,162
-django_postpone_index-0.0.2.dist-info/RECORD,,
+django_postpone_index-0.0.4.dist-info/METADATA,sha256=1ZnvVAhdApfcxVFNJkEKJn0m_r-1kMMHCzGB6xPrTV4,12416
+django_postpone_index-0.0.4.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+django_postpone_index-0.0.4.dist-info/top_level.txt,sha256=B6q_TICqApvruf_NJfnLFNLZLpxMLebQF6cPdKrWm5A,162
+django_postpone_index-0.0.4.dist-info/RECORD,,

postpone_index/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.2'
-__version_tuple__ = version_tuple = (0, 0, 2)
+__version__ = version = '0.0.4'
+__version_tuple__ = version_tuple = (0, 0, 4)
 
 __commit_id__ = commit_id = None

postpone_index/contrib/postgres/schema.py

@@ -25,20 +25,10 @@ class DatabaseSchemaEditorMixin(Utils):
         """
         if self._base_tables_created:
             return
-
         # Avoid circular import
         from postpone_index.models import PostponedSQL
 
-        cursor = self.connection.cursor()
-        cursor.execute("SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = '%s' limit 1" % PostponedSQL._meta.db_table)
-        if cursor.fetchall():
-            self._base_tables_created = True
-            logger.debug('[%s] The PostponedSQL storage already exists', self.connection.alias)
-            return
-        logger.info('[%s] The PostponedSQL storage is absent, creating', self.connection.alias)
-        with open(os.path.join(package_folder, 'sql/start.sql')) as f:
-            sql = f.read()
-        cursor.execute(sql)
+        PostponedSQL.objects.using(self.connection.alias)._create_base_tables()
         self._base_tables_created = True
 
     def _ignore(self):
@@ -128,7 +118,7 @@ class DatabaseSchemaEditorMixin(Utils):
                     logger.info('[%s] Removed all indexes on table %s from postponed', self.connection.alias, table_name)
                 return super().execute(sql, params)
             elif match := self._drop_column_re.fullmatch(str(sql)):
-                # Table dropped cancels all postponed operations related
+                # Column dropped cancels all postponed operations related
                 table_name = match.group('table_nameq') or match.group('table_name')
                 column_name = match.group('column_nameq') or match.group('column_name')
                 if PostponedSQL.objects.using(self.connection.alias).filter(
@@ -139,6 +129,37 @@ class DatabaseSchemaEditorMixin(Utils):
                         self.connection.alias, column_name, table_name
                     )
                 return super().execute(sql, params)
+            elif match := self._rename_column_re.fullmatch(str(sql)):
+                # Rename column changes the postponed indexes to alter SQL but not the index name
+                table_name = match.group('table_nameq') or match.group('table_name')
+                column_name = match.group('column_nameq') or match.group('column_name')
+                ncolumn_name = match.group('ncolumn_nameq') or match.group('ncolumn_name')
+                for postponed_sql in PostponedSQL.objects.using(self.connection.alias).filter(
+                    table=table_name, fields__contains='"%s"' % column_name, done=False
+                ):
+                    logger.info(
+                        '[%s] Reformat index %s SQL to rename column %s to %s of table %s from postponed',
+                        self.connection.alias, postponed_sql.db_index, column_name, ncolumn_name, table_name,
+                    )
+                    postponed_sql.sql = postponed_sql.sql.replace('"%s"' % column_name, '"%s"' % ncolumn_name)
+                    postponed_sql.fields = postponed_sql.fields.replace('"%s"' % column_name, '"%s"' % ncolumn_name)
+                    postponed_sql.save(update_fields=['sql', 'fields'])
+                return super().execute(sql, params)
+            elif match := self._rename_table_re.fullmatch(str(sql)):
+                # Rename table changes the postponed indexes to alter SQL and table name but not anything other
+                table_name = match.group('table_nameq') or match.group('table_name')
+                ntable_name = match.group('ntable_nameq') or match.group('ntable_name')
+                for postponed_sql in PostponedSQL.objects.using(self.connection.alias).filter(
+                    table=table_name, done=False
+                ):
+                    logger.info(
+                        '[%s] Reformat index %s SQL to rename table %s to %s from postponed',
+                        self.connection.alias, postponed_sql.db_index, table_name, ntable_name,
+                    )
+                    postponed_sql.sql = postponed_sql.sql.replace('"%s"' % table_name, '"%s"' % ntable_name)
+                    postponed_sql.table = ntable_name
+                    postponed_sql.save(update_fields=['sql', 'table'])
+                return super().execute(sql, params)
             else:
                 logger.debug('[%s] No special statements: %s', self.connection.alias, sql)
                 return super().execute(sql, params)

postpone_index/models.py

@@ -1,12 +1,79 @@
+import logging
+import os
 import time
 
 from unlimited_char.fields import CharField
 
-from django.db import models
+from django.db import connections, models
 from django.utils.translation import gettext_lazy as _
 
 
+logger = logging.getLogger(__name__)
+package_folder = os.path.dirname(os.path.abspath(__file__))
+
+
+class PostponedSQLQuerySet(models.QuerySet):
+    """
+    QuerySet for PostponedSQL model.
+
+    Redefined for admin view when table is absent.
+    """
+
+    def _create_base_tables(self):
+        """
+        Create base package table if absent
+        """
+        if self._is_present():
+            logger.debug('[%s] The PostponedSQL storage already exists', self.db)
+            return
+        logger.info('[%s] The PostponedSQL storage is absent, creating', self.db)
+        with open(os.path.join(package_folder, 'sql/start.sql')) as f:
+            sql = f.read()
+        cursor = connections[self.db].cursor()
+        cursor.execute(sql)
+
+    def count(self):
+        """
+        Count only if table is present
+        """
+        if not self._is_present():
+            return 0
+        return super().count()
+
+    def _is_present(self):
+        """
+        Check if table is present
+        """
+        cursor = connections[self.db].cursor()
+        cursor.execute(
+            """
+                SELECT 1 FROM information_schema.tables
+                WHERE table_schema = 'public' AND table_name = '%s' limit 1
+            """ % PostponedSQL._meta.db_table
+        )
+        return bool(cursor.fetchall())
+
+
+class PostponedSQLManager(models.Manager):
+    """
+    Manager for PostponedSQL model.
+
+    Redefined for admin view when table is absent.
+    """
+
+    def get_queryset(self):
+        """
+        Get custom QuerySet
+        """
+        qs = PostponedSQLQuerySet(self.model, using=self._db)
+        if not qs._is_present():
+            return qs.none()
+        return qs
+
+
 class PostponedSQL(models.Model):
+    """Model to store postponed SQL commands"""
+
     ts = models.BigIntegerField(
         primary_key=True, editable=False,
         default=time.time_ns,
@@ -47,6 +114,8 @@ class PostponedSQL(models.Model):
         help_text=_('Last error reported when tried to apply')
     )
 
+    objects = PostponedSQLManager()
+
     @property
     def d(self):
         """Status mark"""

postpone_index/utils.py

@@ -26,7 +26,7 @@ class Utils:
         r'^\s*CREATE\s+(?P<unique>UNIQUE\s+)?INDEX\s+(IF\s+NOT\s+EXISTS\s+)?'
         r'(((?P<iq>")?(?P<index_nameq>[^"]+)(?P=iq))|(?P<index_name>[^\s]+))'
         r'\s+ON\s+(?P<only>ONLY\s+)?'
-        r'(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[^\s]+))'
+        r'(("?)public("?)\.)?(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[_a-zA-Z0-9]+))'
         r'(?P<rest>.*)$',
         re.IGNORECASE | re.MULTILINE
     )
@@ -38,13 +38,13 @@ class Utils:
     )
     _drop_table_re = re.compile(
         r'^\s*DROP\s+TABLE\s+(IF\s+EXISTS\s+)?'
-        r'(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[^\s]+))'
+        r'(("?)public("?)\.)?(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[_a-zA-Z0-9]+))'
         r'(?P<rest>.*)$',
         re.IGNORECASE | re.MULTILINE
     )
     _add_constraint_re = re.compile(
         r'^\s*ALTER\s+TABLE\s+'
-        r'(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[^\s]+))'
+        r'(("?)public("?)\.)?(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[_a-zA-Z0-9]+))'
         r'\s+ADD\s+CONSTRAINT\s+'
         r'(((?P<iq>")?(?P<index_nameq>[^"]+)(?P=iq))|(?P<index_name>[^\s]+))'
         r'\s+UNIQUE\s+'
@@ -53,7 +53,7 @@ class Utils:
     )
     _drop_constraint_re = re.compile(
         r'^\s*ALTER\s+TABLE\s+'
-        r'(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[^\s]+))'
+        r'(("?)public("?)\.)?(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[_a-zA-Z0-9]+))'
         r'\s+DROP\s+CONSTRAINT\s+'
         r'(((?P<iq>")?(?P<index_nameq>[^"]+)(?P=iq))|(?P<index_name>[^\s]+))'
         r'(?P<rest>.*)$',
@@ -61,12 +61,30 @@ class Utils:
     )
     _drop_column_re = re.compile(
         r'^\s*ALTER\s+TABLE\s+'
-        r'(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[^\s]+))'
+        r'(("?)public("?)\.)?(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[_a-zA-Z0-9]+))'
         r'\s+DROP\s+COLUMN\s+'
         r'(((?P<cq>")?(?P<column_nameq>[^"]+)(?P=cq))|(?P<column_name>[^\s]+))'
         r'(?P<rest>.*)$',
         re.IGNORECASE | re.MULTILINE
     )
+    _rename_column_re = re.compile(
+        r'^\s*ALTER\s+TABLE\s+'
+        r'(("?)public("?)\.)?(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[_a-zA-Z0-9]+))'
+        r'\s+RENAME\s+COLUMN\s+'
+        r'(((?P<cq>")?(?P<column_nameq>[^"]+)(?P=cq))|(?P<column_name>[^\s]+))'
+        r'\s+TO\s+'
+        r'(((?P<nq>")?(?P<ncolumn_nameq>[^"]+)(?P=nq))|(?P<ncolumn_name>[^\s]+))'
+        r'(?P<rest>.*)$',
+        re.IGNORECASE | re.MULTILINE
+    )
+    _rename_table_re = re.compile(
+        r'^\s*ALTER\s+TABLE\s+'
+        r'(("?)public("?)\.)?(((?P<tq>")?(?P<table_nameq>[^"]+)(?P=tq))|(?P<table_name>[_a-zA-Z0-9]+))'
+        r'\s+RENAME\s+TO\s+'
+        r'(((?P<nq>")?(?P<ntable_nameq>[^"]+)(?P=nq))|(?P<ntable_name>[^\s]+))'
+        r'(?P<rest>.*)$',
+        re.IGNORECASE | re.MULTILINE
+    )
     _column_name_re = re.compile(
         r'(((?P<cq>")(?P<column_nameq>[^"]+)(?P=cq))|(?P<column_name>[^\s]+))',
         re.IGNORECASE | re.MULTILINE