morango 0.6.14__py2.py3-none-any.whl → 0.8.6__py2.py3-none-any.whl

morango/migrations/0012_auto_20180927_1658.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.15 on 2018-09-27 16:58
-from __future__ import unicode_literals
-
 from django.db import migrations
 from django.db import models
 

morango/migrations/0013_auto_20190627_1513.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.21 on 2019-06-27 22:13
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 import morango.models.fields.uuids

morango/migrations/0014_syncsession_extra_fields.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.27 on 2019-12-30 18:28
-from __future__ import unicode_literals
-
 from django.db import migrations
 from django.db import models
 

morango/migrations/0015_auto_20200508_2104.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.29 on 2020-05-08 21:04
-from __future__ import unicode_literals
-
-from django.db import migrations, models
+from django.db import migrations
+from django.db import models
 
 
 class Migration(migrations.Migration):

morango/migrations/0016_store_deserialization_error.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.28 on 2020-06-10 23:48
-from __future__ import unicode_literals
-
-from django.db import migrations, models
+from django.db import migrations
+from django.db import models
 
 
 class Migration(migrations.Migration):

morango/migrations/0017_store_last_transfer_session_id.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.28 on 2021-06-25 23:13
-from __future__ import unicode_literals
-
 from django.db import migrations
+
 import morango.models.fields.uuids
 
 

morango/migrations/0018_auto_20210714_2216.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.29 on 2021-07-14 22:16
-from __future__ import unicode_literals
-
-from django.db import migrations, models
+from django.db import migrations
+from django.db import models
 
 
 class Migration(migrations.Migration):

morango/migrations/0019_auto_20220113_1807.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.29 on 2022-01-13 18:07
-from __future__ import unicode_literals
-
-from django.db import migrations, models
+from django.db import migrations
+from django.db import models
 
 
 class Migration(migrations.Migration):

morango/migrations/0020_postgres_fix_nullable.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.29 on 2022-01-13 18:07
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

morango/migrations/0021_store_partition_index_create.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.29 on 2022-04-27 16:59
-from __future__ import unicode_literals
-
 from django.db import migrations
 from django.db import models
 

morango/migrations/0022_rename_instance_fields.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2023-01-31 18:53
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('morango', '0021_store_partition_index_create'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='syncsession',
+            old_name='client_instance',
+            new_name='client_instance_json',
+        ),
+        migrations.RenameField(
+            model_name='syncsession',
+            old_name='server_instance',
+            new_name='server_instance_json',
+        ),
+    ]

morango/migrations/0023_add_instance_id_fields.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2023-01-31 19:03
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('morango', '0022_rename_instance_fields'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='syncsession',
+            name='client_instance_id',
+            field=models.UUIDField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='syncsession',
+            name='server_instance_id',
+            field=models.UUIDField(blank=True, null=True),
+        ),
+    ]

morango/migrations/0024_auto_20240129_1757.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.23 on 2024-01-29 17:57
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('morango', '0023_add_instance_id_fields'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='certificate',
+            name='level',
+            field=models.PositiveIntegerField(editable=False),
+        ),
+        migrations.AlterField(
+            model_name='certificate',
+            name='lft',
+            field=models.PositiveIntegerField(editable=False),
+        ),
+        migrations.AlterField(
+            model_name='certificate',
+            name='rght',
+            field=models.PositiveIntegerField(editable=False),
+        ),
+    ]

morango/models/__init__.py

@@ -21,9 +21,6 @@ from morango.models.fields import __all__ as fields_all
 from morango.models.fields.crypto import SharedKey
 from morango.models.fields.uuids import UUIDModelMixin
 from morango.models.manager import SyncableModelManager
-from morango.models.morango_mptt import MorangoMPTTModel
-from morango.models.morango_mptt import MorangoMPTTTreeManager
-from morango.models.morango_mptt import MorangoTreeQuerySet
 from morango.models.query import SyncableModelQuerySet
 from morango.registry import syncable_models
 
@@ -41,9 +38,6 @@ __all__ += [
     "SyncableModelManager",
     "SyncableModelQuerySet",
     "syncable_models",
-    "MorangoTreeQuerySet",
-    "MorangoMPTTTreeManager",
-    "MorangoMPTTModel",
     "DatabaseIDModel",
     "InstanceIDModel",
     "SyncSession",

morango/models/certificates.py

@@ -3,18 +3,18 @@
 Each certificate has a ``private_key`` used for signing (child) certificates (thus giving certain permissions)
 and a ``public_key`` used for verifying that a certificate(s) was properly signed.
 """
-from __future__ import unicode_literals
-
 import json
+import logging
 import string
+from contextlib import contextmanager
 
 import mptt.models
 from django.core.management import call_command
+from django.db import connection
 from django.db import models
 from django.db import transaction
+from django.db.utils import OperationalError
 from django.utils import timezone
-from django.utils.six import string_types
-from future.utils import python_2_unicode_compatible
 
 from .fields.crypto import Key
 from .fields.crypto import PrivateKeyField
@@ -27,21 +27,21 @@ from morango.errors import CertificateScopeNotSubset
 from morango.errors import CertificateSignatureInvalid
 from morango.errors import NonceDoesNotExist
 from morango.errors import NonceExpired
+from morango.sync.backends.utils import load_backend
 from morango.utils import _assert
 
 
-@python_2_unicode_compatible
 class Certificate(mptt.models.MPTTModel, UUIDModelMixin):
 
     uuid_input_fields = ("public_key", "profile", "salt")
 
-    parent = models.ForeignKey("Certificate", blank=True, null=True)
+    parent = models.ForeignKey("Certificate", blank=True, null=True, on_delete=models.CASCADE)
 
     # the Morango profile with which this certificate is associated
     profile = models.CharField(max_length=20)
 
     # scope of this certificate, and version of the scope, along with associated params
-    scope_definition = models.ForeignKey("ScopeDefinition")
+    scope_definition = models.ForeignKey("ScopeDefinition", on_delete=models.CASCADE)
     scope_version = models.IntegerField()
     scope_params = (
         models.TextField()
@@ -199,7 +199,7 @@ class Certificate(mptt.models.MPTTModel, UUIDModelMixin):
     def save_certificate_chain(cls, cert_chain, expected_last_id=None):
 
         # parse the chain from json if needed
-        if isinstance(cert_chain, string_types):
+        if isinstance(cert_chain, str):
             cert_chain = json.loads(cert_chain)
 
         # start from the bottom of the chain
@@ -251,6 +251,37 @@ class Certificate(mptt.models.MPTTModel, UUIDModelMixin):
     def get_scope(self):
         return self.scope_definition.get_scope(self.scope_params)
 
+    @contextmanager
+    def _attempt_lock_mptt(self):
+        from morango.sync.utils import lock_partitions
+
+        DBBackend = load_backend(connection)
+
+        with transaction.atomic():
+            # Call get_root on the parent as it is already saved in the DB
+            root_id = self.parent.get_root().id if self.parent else self.id
+
+            # lock the partitions in our scope to prevent MPTT tree corruption during concurrent certificate creation
+            lock_partitions(DBBackend, sync_filter=Filter(root_id) if root_id else None)
+            yield
+
+    @contextmanager
+    def _lock_mptt(self):
+        try:
+            with self._attempt_lock_mptt():
+                yield
+        except OperationalError as e:
+            if "deadlock detected" in e.args[0]:
+                logging.error("Deadlock detected when attempting to lock MPTT partitions, retrying once more")
+                with self._attempt_lock_mptt():
+                    yield
+            else:
+                raise
+
+    def save(self, *args, **kwargs):
+        with self._lock_mptt():
+            super().save(*args, **kwargs)
+
     def __str__(self):
         if self.scope_definition:
             return self.scope_definition.get_description(self.scope_params)
@@ -324,52 +355,95 @@ class ScopeDefinition(models.Model):
         return Scope(definition=self, params=params)
 
     def get_description(self, params):
-        if isinstance(params, string_types):
+        if isinstance(params, str):
             params = json.loads(params)
         return string.Template(self.description).safe_substitute(params)
 
 
-@python_2_unicode_compatible
 class Filter(object):
-    def __init__(self, template, params={}):
-        # ensure params have been deserialized
-        if isinstance(params, string_types):
-            params = json.loads(params)
-        self._template = template
-        self._params = params
-        self._filter_string = string.Template(template).safe_substitute(params)
-        self._filter_tuple = tuple(self._filter_string.split()) or ("",)
+    def __init__(self, filter_str, params=None):
+        """
+        :param filter_str: The partition filter string, which may have multiple separated by newlines
+        :type filter_str: str
+        :param params: DEPRECATED: USE Filter.from_template() INSTEAD
+        :type params: dict|str
+        """
+        if params is not None:
+            logging.warning("DEPRECATED: Constructing a filter with a template and params is deprecated. Use Filter.from_template() instead")
+            filter_str = str(Filter.from_template(filter_str, params=params))
+
+        self._filter_tuple = tuple(filter_str.split()) or ("",)
 
     def is_subset_of(self, other):
-        for partition in self._filter_tuple:
-            if not partition.startswith(other._filter_tuple):
+        """
+        :param other: The other Filter
+        :type other: Filter
+        :return: A boolean on whether this Filter is captured within the other Filter
+        :rtype: bool
+        """
+        for partition in self:
+            if not other.contains_partition(partition):
                 return False
         return True
 
     def contains_partition(self, partition):
+        """Returns True if the partition starts with as least one of the partitions in this Filter"""
         return partition.startswith(self._filter_tuple)
 
+    def contains_exact_partition(self, partition):
+        """Returns True if the partition exactly matches one of the partitions in this Filter"""
+        return partition in self._filter_tuple
+
+    def copy(self):
+        return Filter(str(self))
+
     def __le__(self, other):
+        """Returns True if this Filter is a subset of the other"""
         return self.is_subset_of(other)
 
     def __eq__(self, other):
+        """Returns True if this Filter has exactly the same partitions as the other"""
         if other is None:
             return False
-        for partition in self._filter_tuple:
-            if partition not in other._filter_tuple:
+        for partition in self:
+            if not other.contains_exact_partition(partition):
                 return False
-        for partition in other._filter_tuple:
-            if partition not in self._filter_tuple:
+        for partition in other:
+            if not self.contains_exact_partition(partition):
                 return False
         return True
 
     def __contains__(self, partition):
+        """
+        Performs a 'startswith' comparison on the partition, determining whether it matches or
+        is a subset of any partition in this Filter
+
+        :param partition: str
+        :return: A boolean
+        :rtype: bool
+        """
         return self.contains_partition(partition)
 
     def __add__(self, other):
-        return Filter(self._filter_string + "\n" + other._filter_string)
+        """
+        The Filter's addition operator overload
+        :param other: Filter or None
+        :type other: Filter|None
+        :return: The combined Filter
+        :rtype: Filter
+        """
+        if other is None:
+            return self
+        # create a list of partition filters, deduplicating them between the two filter objects
+        partitions = []
+        partitions.extend(p for p in self if p)
+        partitions.extend(p for p in other if p and p not in partitions)
+        return Filter("\n".join(partitions))
 
     def __iter__(self):
+        """
+        :rtype: tuple[str]
+        """
         return iter(self._filter_tuple)
 
     def __str__(self):
@@ -378,13 +452,48 @@ class Filter(object):
     def __len__(self):
         return len(self._filter_tuple)
 
+    @classmethod
+    def add(cls, filter_a, filter_b):
+        """
+        The Filter's addition operator overload is already defensive against None being the
+        right-hand operand, but this method is defensive against None being the left-hand operand
+
+        :param filter_a: A Filter or None
+        :type filter_a: Filter|None
+        :param filter_b: A Filter or None
+        :type filter_b: Filter|None
+        :return: The combined Filter or None
+        :rtype: Filter|None
+        """
+        if filter_a is None:
+            return filter_b
+        return filter_a + filter_b
+
+    @classmethod
+    def from_template(cls, template, params=None):
+        """
+        Create a filter from a string template, which may have params that will be replaced with
+        values passed to `params`
+
+        :param template: The partition filter template
+        :type template: str
+        :param params: The param dictionary or JSON object string
+        :type params: dict|str
+        :return: The filter with params replaced
+        :rtype: Filter
+        """
+        if isinstance(params, str):
+            params = json.loads(params)
+        params = params or {}
+        return Filter(string.Template(template).safe_substitute(params))
+
 
 class Scope(object):
     def __init__(self, definition, params):
         # turn the scope definition filter templates into Filter objects
-        rw_filter = Filter(definition.read_write_filter_template, params)
-        self.read_filter = rw_filter + Filter(definition.read_filter_template, params)
-        self.write_filter = rw_filter + Filter(definition.write_filter_template, params)
+        rw_filter = Filter.from_template(definition.read_write_filter_template, params)
+        self.read_filter = rw_filter + Filter.from_template(definition.read_filter_template, params)
+        self.write_filter = rw_filter + Filter.from_template(definition.write_filter_template, params)
 
     def is_subset_of(self, other):
         if not self.read_filter.is_subset_of(other.read_filter):

morango/models/core.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import functools
 import json
 import logging
@@ -17,13 +15,13 @@ from django.db.models import F
 from django.db.models import Func
 from django.db.models import Max
 from django.db.models import Q
+from django.db.models import signals
 from django.db.models import TextField
 from django.db.models import Value
 from django.db.models.deletion import Collector
 from django.db.models.expressions import CombinedExpression
 from django.db.models.fields.related import ForeignKey
 from django.db.models.functions import Cast
-from django.utils import six
 from django.utils import timezone
 from django.utils.functional import cached_property
 
@@ -38,7 +36,6 @@ from morango.models.fields.uuids import UUIDField
 from morango.models.fields.uuids import UUIDModelMixin
 from morango.models.fsic_utils import remove_redundant_instance_counters
 from morango.models.manager import SyncableModelManager
-from morango.models.morango_mptt import MorangoMPTTModel
 from morango.models.utils import get_0_4_system_parameters
 from morango.models.utils import get_0_5_mac_address
 from morango.models.utils import get_0_5_system_id
@@ -240,9 +237,12 @@ class SyncSession(models.Model):
     client_ip = models.CharField(max_length=100, blank=True)
     server_ip = models.CharField(max_length=100, blank=True)
 
-    # serialized copies of the client and server instance model fields, for debugging/tracking purposes
-    client_instance = models.TextField(default="{}")
-    server_instance = models.TextField(default="{}")
+    # track the instance IDs for convenient querying, as well as serialized copies of
+    # the client and server instance model fields for debugging/tracking purposes
+    client_instance_id = models.UUIDField(null=True, blank=True)
+    client_instance_json = models.TextField(default="{}")
+    server_instance_id = models.UUIDField(null=True, blank=True)
+    server_instance_json = models.TextField(default="{}")
 
     # used to store other data we may need to know about this sync session
     extra_fields = models.TextField(default="{}")
@@ -252,11 +252,11 @@ class SyncSession(models.Model):
 
     @cached_property
     def client_instance_data(self):
-        return json.loads(self.client_instance)
+        return json.loads(self.client_instance_json)
 
     @cached_property
     def server_instance_data(self):
-        return json.loads(self.server_instance)
+        return json.loads(self.server_instance_json)
 
 
 class TransferSession(models.Model):
@@ -351,10 +351,10 @@ class TransferSession(models.Model):
 
     def get_touched_record_ids_for_model(self, model):
         if isinstance(model, SyncableModel) or (
-            isinstance(model, six.class_types) and issubclass(model, SyncableModel)
+            isinstance(model, type) and issubclass(model, SyncableModel)
         ):
             model = model.morango_model_name
-        _assert(isinstance(model, six.string_types), "Model must resolve to string")
+        _assert(isinstance(model, str), "Model must resolve to string")
         return Store.objects.filter(
             model_name=model, last_transfer_session_id=self.id
         ).values_list("id", flat=True)
@@ -418,7 +418,7 @@ class StoreQueryset(models.QuerySet):
             self.annotate(id_cast=Cast("id", TextField()))
             # remove dashes from char uuid
             .annotate(
-                fixed_id=Func(F("id_cast"), Value("-"), Value(""), function="replace")
+                fixed_id=Func(F("id_cast"), Value("-"), Value(""), function="replace", output_field=TextField())
             )
             # return as list
             .values_list("fixed_id", flat=True)
@@ -450,6 +450,7 @@ class Store(AbstractStore):
     class Meta:
         indexes = [
             models.Index(fields=["partition"], name="idx_morango_store_partition"),
+            models.Index(fields=["profile", "model_name", "partition", "dirty_bit"], condition=models.Q(dirty_bit=True), name="idx_morango_deserialize"),
         ]
 
     def _deserialize_store_model(self, fk_cache, defer_fks=False):  # noqa: C901
@@ -465,14 +466,14 @@ class Store(AbstractStore):
         klass_model = syncable_models.get_model(self.profile, self.model_name)
         # if store model marked as deleted, attempt to delete in app layer
         if self.deleted:
-            # if hard deleted, propagate to related models
-            if self.hard_deleted:
-                try:
-                    klass_model.objects.get(id=self.id).delete(hard_delete=True)
-                except klass_model.DoesNotExist:
-                    pass
-            else:
-                klass_model.objects.filter(id=self.id).delete()
+            # Don't differentiate between deletion and hard deletion here,
+            # as we don't want to add additional tracking for models in either case,
+            # just to actually delete them.
+            # Import here to avoid circular import, as the utils module
+            # imports core models.
+            from morango.sync.utils import mute_signals
+            with mute_signals(signals.post_delete):
+                klass_model.syncing_objects.filter(id=self.id).delete()
             return None, deferred_fks
         else:
             # load model into memory
@@ -608,7 +609,7 @@ class DatabaseMaxCounter(AbstractCounter):
                         )
         else:
             updated_fsic = {}
-            for key, value in six.iteritems(fsics):
+            for key, value in fsics.items():
                 if key in internal_fsic:
                     # if same instance id, update fsic with larger value
                     if fsics[key] > internal_fsic[key]:
@@ -618,7 +619,7 @@ class DatabaseMaxCounter(AbstractCounter):
                     updated_fsic[key] = fsics[key]
 
             # load database max counters
-            for (key, value) in six.iteritems(updated_fsic):
+            for (key, value) in updated_fsic.items():
                 for f in sync_filter:
                     DatabaseMaxCounter.objects.update_or_create(
                         instance_id=key, partition=f, defaults={"counter": value}
@@ -805,6 +806,11 @@ class SyncableModel(UUIDModelMixin):
 
     objects = SyncableModelManager()
 
+    # Add a special syncing_objects queryset to every SyncableModel for use in syncing operations.
+    # This means that we still deal with the entire set of objects when syncing, even if the default
+    # model manager has been overridden to filter the queryset.
+    syncing_objects = SyncableModelManager()
+
     class Meta:
         abstract = True
 
@@ -841,10 +847,8 @@ class SyncableModel(UUIDModelMixin):
         with transaction.atomic():
             if hard_delete:
                 # set hard deletion for all related models
-                for model, instances in six.iteritems(collector.data):
-                    if issubclass(model, SyncableModel) or issubclass(
-                        model, MorangoMPTTModel
-                    ):
+                for model, instances in collector.data.items():
+                    if issubclass(model, SyncableModel):
                         for obj in instances:
                             obj._update_hard_deleted_models()
             return collector.delete()
@@ -928,15 +932,8 @@ class SyncableModel(UUIDModelMixin):
                 continue
             if f.attname in self._morango_internal_fields_not_to_serialize:
                 continue
-            # case if model is morango mptt
-            if f.attname in getattr(
-                self,
-                "_internal_mptt_fields_not_to_serialize",
-                "_internal_fields_not_to_serialize",
-            ):
-                continue
-            if hasattr(f, "value_from_object_json_compatible"):
-                data[f.attname] = f.value_from_object_json_compatible(self)
+            if getattr(f, "morango_serialize_to_string", False):
+                data[f.attname] = f.value_to_string(self)
             else:
                 data[f.attname] = f.value_from_object(self)
         return data
@@ -947,7 +944,10 @@ class SyncableModel(UUIDModelMixin):
         kwargs = {}
         for f in cls._meta.concrete_fields:
             if f.attname in dict_model:
-                kwargs[f.attname] = dict_model[f.attname]
+                if getattr(f, "morango_serialize_to_string", False):
+                    kwargs[f.attname] = f.to_python(dict_model[f.attname])
+                else:
+                    kwargs[f.attname] = dict_model[f.attname]
         return cls(**kwargs)
 
     @classmethod