django-postpone-index 0.0.1__tar.gz

django_postpone_index-0.0.1/.github/workflows/ci.yml

@@ -0,0 +1,147 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - "**"
+    tags:
+      - "v*"
+      - "*.*.*"
+      - "*.*.*.*"
+  pull_request:
+
+jobs:
+  syntax:
+    name: tox (syntax)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install build tools
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade tox
+
+      - name: Run tox
+        run: |
+          tox run -e syntax
+
+  test:
+    name: tox (-f ${{ matrix.tox_factor }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - python-version: "3.10"
+            tox_factor: py310
+          - python-version: "3.11"
+            tox_factor: py311
+          - python-version: "3.12"
+            tox_factor: py312
+          - python-version: "3.13"
+            tox_factor: py313
+          - python-version: "3.14"
+            tox_factor: py3.14
+
+    services:
+      postgres:
+        image: postgis/postgis:15-3.4
+        env:
+          POSTGRES_USER: test
+          POSTGRES_PASSWORD: test
+          POSTGRES_DB: postpone_index
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U test -d postpone_index"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+          cache: pip
+
+      - name: Install build tools
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade tox
+
+      - name: Install necessary libraries
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gdal-bin libgdal-dev libgeos-dev
+
+      - name: Prepare databases
+        env:
+          PGPASSWORD: test
+        run: |
+          until pg_isready -h 127.0.0.1 -p 5432 -U test -d postpone_index; do sleep 1; done
+          createdb -h 127.0.0.1 -p 5432 -U test postpone_index_additional
+
+      - name: Run tox
+        run: |
+          tox run -f ${{ matrix.tox_factor }}
+
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: [syntax, test]
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Determine if this is a release tag
+        id: release
+        env:
+          TAG_NAME: ${{ github.ref_name }}
+        run: |
+          python - <<'PY'
+          import os, re
+          tag = os.environ.get("TAG_NAME", "")
+          is_release = bool(re.match(r"^(?:v)?\d+\.\d+\.\d+(?:\.\d+)?$", tag))
+          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
+              f.write(f"is_release={str(is_release).lower()}\n")
+          print(f"tag={tag} is_release={is_release}")
+          PY
+
+      - name: Build distributions
+        if: steps.release.outputs.is_release == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade build
+          python -m build
+
+      - name: Publish distributions to PyPI
+        if: steps.release.outputs.is_release == 'true'
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

django_postpone_index-0.0.1/.gitignore

@@ -0,0 +1,29 @@
+docs/_build
+build/
+dist/
+*.egg-info
+locale
+*.pyc
+*.pyo
+*.sqlite*
+lcov.info
+benchmarks/results/
+coverage.xml
+.coverage
+htmlcov/
+.svn/
+.tox/
+.vscode/
+.venv/
+.clangd
+__pycache__/
+.ropeproject
+polib.py
+reversion
+tinymce
+pip-log.txt
+MANIFEST
+dist
+
+# Generated by setuptools-scm during builds
+postpone_index/_version.py

django_postpone_index-0.0.1/PKG-INFO

@@ -0,0 +1,276 @@
+Metadata-Version: 2.4
+Name: django-postpone-index
+Version: 0.0.1
+Summary: Postpone index creation to provide Zero Downtime Migration feature
+Home-page: https://github.com/nnseva/django-postpone-index
+Author: Vsevolod Novikov
+Author-email: nnseva@gmail.com
+License: LGPLv3
+Project-URL: Source, https://github.com/nnseva/django-postpone-index
+Project-URL: Issues, https://github.com/nnseva/django-postpone-index/issues
+Keywords: zero-downtime-migration,django,migration,zero-downtime,downtime,postgres,postgresql
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Framework :: Django
+Classifier: Framework :: Django :: 2.2
+Classifier: Framework :: Django :: 3.0
+Classifier: Framework :: Django :: 3.1
+Classifier: Framework :: Django :: 3.2
+Classifier: Framework :: Django :: 4.0
+Classifier: Framework :: Django :: 4.1
+Classifier: Framework :: Django :: 4.2
+Classifier: Framework :: Django :: 5.0
+Classifier: Framework :: Django :: 5.1
+Classifier: Framework :: Django :: 5.2
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: django2-postgres-backport
+Requires-Dist: django-unlimited-char
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+[![Tests](https://github.com/nnseva/django-postpone-index/actions/workflows/ci.yml/badge.svg)](https://github.com/nnseva/django-postpone-index/actions/workflows/ci.yml)
+
+# Django Postpone Index
+
+This package provides modules and tools to postpone any index creation instead doing it inside the migration,
+to provide *Zero Downtime Migration* feature.
+
+The package is now using the PostgresSQL-specific `CREATE INDEX CONCURRENTLY` SQL command, so is applicable
+only to the PostgreSQL backend.
+
+## Installation
+
+*Stable version* from the PyPi package repository
+```bash
+pip install django-postpone-index
+```
+
+*Last development version* from the GitHub source version control system
+```
+pip install git+git://github.com/nnseva/django-postpone-index.git
+```
+
+## Problem Description
+
+Large data leads to long index creation time.
+
+When the migration is automatically created, it executes all SQL commands creating index inside a transaction.
+
+Large data and index creation inside a transaction lead to long-term table lock which blocks any data writting to the table.
+
+On the other side, `CREATE INDEX CONCURRENTLY` SQL command may solve the problem, but this SQL command can not be executed inside a transaction block.
+
+The `AddIndexConcurrently` might be created in a separate migration, moving out the automatically generated `AddIndex` from the migration,
+but not all indexes are created using `AddIndex`.
+
+## Solution
+
+All index creation SQL commands (as well as unique constraints creation) are catched
+and postponed using a special `PostponedSQL` model (the `DROP INDEX` and `DROP CONSTRAINT` SQL commands
+are still executed immediately).
+
+When the migration is finished, the postponed indexes may be created in a separate process
+using `CREATE INDEX CONCURRENTLY` SQL command by the `apply_postponed` management command.
+Apart from the standard migration, this process doesn't lock the whole table for a long time.
+
+Failed index creation statements don't lead to the command failure
+(until a special command line parameter passed). Every failed statement is stored
+as erroneous instead. When the data is fixed, you can execute the `apply_postponed` management
+command again to restore the failed indexes.
+
+## Complex Use Cases
+
+The following complex use cases are processed by the package.
+
+- Several create/drop pairs. There can be several create/drop index pairs if several migrations applied at once.
+- Back Migration. The both, forward and backward migrations are processed.
+- Implicit index drop while removing the table. The Django doesn't issue a separate SQL to drop indexes of the dropped table.
+- Implicit index drop while removing the field. The Django doesn't issue a separate SQL to drop indexes related to the dropped column.
+
+## Using
+
+Include the `postpone_index` application in `setting.py`:
+
+```python
+INSTALLED_APPS = [
+    ...
+    'postpone_index',
+    ...
+]
+```
+
+Use `pospone_index.contrib.postgres` or `postpone_index.contrib.postgis` engines instead of the Django-provided in `settings.py`:
+
+```python
+DATABASES = {
+    'default': {
+        'ENGINE': 'postpone_index.contrib.postgres',
+        ...
+    }
+}
+```
+
+If you provide your own database engine instead of the Django-provided, you can also
+combine `pospone_index.contrib.postgres.schema.DatabaseSchemaEditorMixin` with your own Database Schema Editor, f.e.:
+
+`mybackend/schema.py`
+```python
+from django.db.backends.postgresql.schema import DatabaseSchemaEditor as _DatabaseSchemaEditor
+from pospone_index.contrib.postgres.schema import DatabaseSchemaEditorMixin
+
+class PostponeIndexDatabaseSchemaEditor(DatabaseSchemaEditorMixin, _DatabaseSchemaEditor):
+    # Your own code
+    ...
+```
+
+`mybackend/base.py`
+```python
+from django.db.backends.postgresql.base import (
+    DatabaseWrapper as _DatabaseWrapper,
+)
+
+from mybackend.schema import PostponeIndexDatabaseSchemaEditor
+
+
+class DatabaseWrapper(_DatabaseWrapper):
+    """Database wrapper"""
+
+    SchemaEditorClass = PostponeIndexDatabaseSchemaEditor
+    # Your own code
+    ...
+```
+
+Execute `apply_postponed` management command every time after the `migrate` management command to create new postponed indexes.
+
+Monitor `PostponedSQL` model instances to see errors on the SQL execution.
+
+After the data is fixed, you can try to recreate the postponed invalid indexes just
+calling the `apply_postponed` migration command again. All not-applied indexes will be tried to create again.
+
+**NOTICE** the `apply_postponed` management command doesn't have any explicit locking mechanics. Avoid starting this
+command concurrently with itself or another `migrate` command on the same database.
+
+## Django testing
+
+Django migrates testing database before tests. Always use `POSTPONE_INDEX_IGNORE = True` settings to avoid postpone index
+for the testing database.
+
+If you want to check your own migration with the postpone index switched on,
+use the `postpone_index.testing_utils.TestCase` and `override_settings` Django feature with the following trick:
+
+```python
+from django.core.management import call_command
+from django.test import override_settings
+from postpone_index.models import PostponedSQL
+from postpone_index import testing_utils
+
+class ModuleTest(testing_utils.TestCase):
+    # Notice that the base TestCase is TransactionalTestCase
+
+    @classmethod
+    def setUpClass(cls):
+        # If you want to have customized setUpClass, call the method of the base class
+        super().setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        # If you want to have customized tearDownClass, call the method of the base class
+        super().tearDownClass()
+
+    def test_my_special_migration_case(self):
+        """Explicitly check my migration with postpone_index"""
+
+        module_to_check = "my_module"           # Your Django App
+        migration_before_the_check = "0005"     # Just before your migration
+        migration_to_check = "0006"             # The migration you check
+
+        # Notice that POSTPONED_INDEX_IGNORE is True by default while testing
+        call_command('migrate', module_to_check, migration_before_the_check)
+
+        with override_settings(
+            POSTPONE_INDEX_IGNORE=False
+        ):
+            # Here we can check how it's going with `postpone_index` activated
+
+            # Check whether your migration works as expected with postponed indexes
+            call_command('migrate', module_to_check, migration_to_check)
+
+            # Here you can check how the module works before apply_postponed
+            ...
+
+            # Check whether the indexes applied properly. The `-x` parameter
+            # causes exception on errors
+            call_command('apply_postponed', 'run', '-x')
+```
+
+## Django settings
+
+### `POSTPONE_INDEX_IGNORE`
+
+The setting totally switches off the functionality of the package.
+
+Always use this setting in the test environment to avoid using postponed index creation for the test database.
+
+May be used in a heterogeneous database environment to switch off the package functionality on unsupported databases.
+
+### `POSTPONE_INDEX_ADMIN_IGNORE`
+
+The `PostponedSQL` model admin view is switched on by default. You can totally switch it off,
+or create your own admin class instead. Use `postpone_index.admin.PostponedSQLAdminMixin` as a base class if necessary.
+
+## Django database
+
+The Django supports heterogeneous database environment in a single project. Every single database has it's own
+state of migrations executed by the `manage.py migrate --database <alias>`.
+
+The `apply_postponed` command also supports selection of the database alias using similar syntax:
+
+```bash
+
+# The 'default' database alias is used as a default
+python manage.py migrate
+python manage.py apply_postponed
+
+# A non-default database alias parameter has similar syntax
+python manage.py migrate --database another-postgres-database
+python manage.py apply_postponed --database another-postgres-database
+```
+
+Use `POSTPONE_INDEX_IGNORE=1` environment to switch off the package functionality on migrations running on unsupported database engines like:
+
+```bash
+POSTPONE_INDEX_IGNORE=1 python manage.py migrate --database non-postgres-database
+```
+
+## Special migrations to avoid postpone index
+
+Sometimes you may need to avoid the `postpone_index` applied to a single migration.
+
+Just include the `PostponeIndexIgnoreMigrationMixin` into a base class list for your special migration:
+
+```python
+from django.db import migrations, models
+from postpone_index.migration_utils import PostponeIndexIgnoreMigrationMixin
+
+class Migration(PostponeIndexIgnoreMigrationMixin, migrations.Migration):
+    ...
+```

django_postpone_index-0.0.1/README.md

@@ -0,0 +1,227 @@
+[![Tests](https://github.com/nnseva/django-postpone-index/actions/workflows/ci.yml/badge.svg)](https://github.com/nnseva/django-postpone-index/actions/workflows/ci.yml)
+
+# Django Postpone Index
+
+This package provides modules and tools to postpone any index creation instead doing it inside the migration,
+to provide *Zero Downtime Migration* feature.
+
+The package is now using the PostgresSQL-specific `CREATE INDEX CONCURRENTLY` SQL command, so is applicable
+only to the PostgreSQL backend.
+
+## Installation
+
+*Stable version* from the PyPi package repository
+```bash
+pip install django-postpone-index
+```
+
+*Last development version* from the GitHub source version control system
+```
+pip install git+git://github.com/nnseva/django-postpone-index.git
+```
+
+## Problem Description
+
+Large data leads to long index creation time.
+
+When the migration is automatically created, it executes all SQL commands creating index inside a transaction.
+
+Large data and index creation inside a transaction lead to long-term table lock which blocks any data writting to the table.
+
+On the other side, `CREATE INDEX CONCURRENTLY` SQL command may solve the problem, but this SQL command can not be executed inside a transaction block.
+
+The `AddIndexConcurrently` might be created in a separate migration, moving out the automatically generated `AddIndex` from the migration,
+but not all indexes are created using `AddIndex`.
+
+## Solution
+
+All index creation SQL commands (as well as unique constraints creation) are catched
+and postponed using a special `PostponedSQL` model (the `DROP INDEX` and `DROP CONSTRAINT` SQL commands
+are still executed immediately).
+
+When the migration is finished, the postponed indexes may be created in a separate process
+using `CREATE INDEX CONCURRENTLY` SQL command by the `apply_postponed` management command.
+Apart from the standard migration, this process doesn't lock the whole table for a long time.
+
+Failed index creation statements don't lead to the command failure
+(until a special command line parameter passed). Every failed statement is stored
+as erroneous instead. When the data is fixed, you can execute the `apply_postponed` management
+command again to restore the failed indexes.
+
+## Complex Use Cases
+
+The following complex use cases are processed by the package.
+
+- Several create/drop pairs. There can be several create/drop index pairs if several migrations applied at once.
+- Back Migration. The both, forward and backward migrations are processed.
+- Implicit index drop while removing the table. The Django doesn't issue a separate SQL to drop indexes of the dropped table.
+- Implicit index drop while removing the field. The Django doesn't issue a separate SQL to drop indexes related to the dropped column.
+
+## Using
+
+Include the `postpone_index` application in `setting.py`:
+
+```python
+INSTALLED_APPS = [
+    ...
+    'postpone_index',
+    ...
+]
+```
+
+Use `pospone_index.contrib.postgres` or `postpone_index.contrib.postgis` engines instead of the Django-provided in `settings.py`:
+
+```python
+DATABASES = {
+    'default': {
+        'ENGINE': 'postpone_index.contrib.postgres',
+        ...
+    }
+}
+```
+
+If you provide your own database engine instead of the Django-provided, you can also
+combine `pospone_index.contrib.postgres.schema.DatabaseSchemaEditorMixin` with your own Database Schema Editor, f.e.:
+
+`mybackend/schema.py`
+```python
+from django.db.backends.postgresql.schema import DatabaseSchemaEditor as _DatabaseSchemaEditor
+from pospone_index.contrib.postgres.schema import DatabaseSchemaEditorMixin
+
+class PostponeIndexDatabaseSchemaEditor(DatabaseSchemaEditorMixin, _DatabaseSchemaEditor):
+    # Your own code
+    ...
+```
+
+`mybackend/base.py`
+```python
+from django.db.backends.postgresql.base import (
+    DatabaseWrapper as _DatabaseWrapper,
+)
+
+from mybackend.schema import PostponeIndexDatabaseSchemaEditor
+
+
+class DatabaseWrapper(_DatabaseWrapper):
+    """Database wrapper"""
+
+    SchemaEditorClass = PostponeIndexDatabaseSchemaEditor
+    # Your own code
+    ...
+```
+
+Execute `apply_postponed` management command every time after the `migrate` management command to create new postponed indexes.
+
+Monitor `PostponedSQL` model instances to see errors on the SQL execution.
+
+After the data is fixed, you can try to recreate the postponed invalid indexes just
+calling the `apply_postponed` migration command again. All not-applied indexes will be tried to create again.
+
+**NOTICE** the `apply_postponed` management command doesn't have any explicit locking mechanics. Avoid starting this
+command concurrently with itself or another `migrate` command on the same database.
+
+## Django testing
+
+Django migrates testing database before tests. Always use `POSTPONE_INDEX_IGNORE = True` settings to avoid postpone index
+for the testing database.
+
+If you want to check your own migration with the postpone index switched on,
+use the `postpone_index.testing_utils.TestCase` and `override_settings` Django feature with the following trick:
+
+```python
+from django.core.management import call_command
+from django.test import override_settings
+from postpone_index.models import PostponedSQL
+from postpone_index import testing_utils
+
+class ModuleTest(testing_utils.TestCase):
+    # Notice that the base TestCase is TransactionalTestCase
+
+    @classmethod
+    def setUpClass(cls):
+        # If you want to have customized setUpClass, call the method of the base class
+        super().setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        # If you want to have customized tearDownClass, call the method of the base class
+        super().tearDownClass()
+
+    def test_my_special_migration_case(self):
+        """Explicitly check my migration with postpone_index"""
+
+        module_to_check = "my_module"           # Your Django App
+        migration_before_the_check = "0005"     # Just before your migration
+        migration_to_check = "0006"             # The migration you check
+
+        # Notice that POSTPONED_INDEX_IGNORE is True by default while testing
+        call_command('migrate', module_to_check, migration_before_the_check)
+
+        with override_settings(
+            POSTPONE_INDEX_IGNORE=False
+        ):
+            # Here we can check how it's going with `postpone_index` activated
+
+            # Check whether your migration works as expected with postponed indexes
+            call_command('migrate', module_to_check, migration_to_check)
+
+            # Here you can check how the module works before apply_postponed
+            ...
+
+            # Check whether the indexes applied properly. The `-x` parameter
+            # causes exception on errors
+            call_command('apply_postponed', 'run', '-x')
+```
+
+## Django settings
+
+### `POSTPONE_INDEX_IGNORE`
+
+The setting totally switches off the functionality of the package.
+
+Always use this setting in the test environment to avoid using postponed index creation for the test database.
+
+May be used in a heterogeneous database environment to switch off the package functionality on unsupported databases.
+
+### `POSTPONE_INDEX_ADMIN_IGNORE`
+
+The `PostponedSQL` model admin view is switched on by default. You can totally switch it off,
+or create your own admin class instead. Use `postpone_index.admin.PostponedSQLAdminMixin` as a base class if necessary.
+
+## Django database
+
+The Django supports heterogeneous database environment in a single project. Every single database has it's own
+state of migrations executed by the `manage.py migrate --database <alias>`.
+
+The `apply_postponed` command also supports selection of the database alias using similar syntax:
+
+```bash
+
+# The 'default' database alias is used as a default
+python manage.py migrate
+python manage.py apply_postponed
+
+# A non-default database alias parameter has similar syntax
+python manage.py migrate --database another-postgres-database
+python manage.py apply_postponed --database another-postgres-database
+```
+
+Use `POSTPONE_INDEX_IGNORE=1` environment to switch off the package functionality on migrations running on unsupported database engines like:
+
+```bash
+POSTPONE_INDEX_IGNORE=1 python manage.py migrate --database non-postgres-database
+```
+
+## Special migrations to avoid postpone index
+
+Sometimes you may need to avoid the `postpone_index` applied to a single migration.
+
+Just include the `PostponeIndexIgnoreMigrationMixin` into a base class list for your special migration:
+
+```python
+from django.db import migrations, models
+from postpone_index.migration_utils import PostponeIndexIgnoreMigrationMixin
+
+class Migration(PostponeIndexIgnoreMigrationMixin, migrations.Migration):
+    ...
+```