morango 0.6.11__py2.py3-none-any.whl → 0.8.6__py2.py3-none-any.whl

morango/models/core.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import functools
 import json
 import logging
@@ -17,13 +15,13 @@ from django.db.models import F
 from django.db.models import Func
 from django.db.models import Max
 from django.db.models import Q
+from django.db.models import signals
 from django.db.models import TextField
 from django.db.models import Value
 from django.db.models.deletion import Collector
 from django.db.models.expressions import CombinedExpression
 from django.db.models.fields.related import ForeignKey
 from django.db.models.functions import Cast
-from django.utils import six
 from django.utils import timezone
 from django.utils.functional import cached_property
 
@@ -38,7 +36,6 @@ from morango.models.fields.uuids import UUIDField
 from morango.models.fields.uuids import UUIDModelMixin
 from morango.models.fsic_utils import remove_redundant_instance_counters
 from morango.models.manager import SyncableModelManager
-from morango.models.morango_mptt import MorangoMPTTModel
 from morango.models.utils import get_0_4_system_parameters
 from morango.models.utils import get_0_5_mac_address
 from morango.models.utils import get_0_5_system_id
@@ -122,7 +119,7 @@ class InstanceIDModel(models.Model):
     hostname = models.TextField()
     sysversion = models.TextField()
     node_id = models.CharField(max_length=20, blank=True)
-    database = models.ForeignKey(DatabaseIDModel)
+    database = models.ForeignKey(DatabaseIDModel, on_delete=models.CASCADE)
     counter = models.IntegerField(default=0)
     current = models.BooleanField(default=True)
     db_path = models.CharField(max_length=1000)
@@ -219,10 +216,10 @@ class SyncSession(models.Model):
 
     # track the certificates being used by each side for this session
     client_certificate = models.ForeignKey(
-        Certificate, blank=True, null=True, related_name="syncsessions_client"
+        Certificate, blank=True, null=True, related_name="syncsessions_client", on_delete=models.CASCADE
     )
     server_certificate = models.ForeignKey(
-        Certificate, blank=True, null=True, related_name="syncsessions_server"
+        Certificate, blank=True, null=True, related_name="syncsessions_server", on_delete=models.CASCADE
     )
 
     # track the morango profile this sync session is happening for
@@ -240,9 +237,12 @@ class SyncSession(models.Model):
     client_ip = models.CharField(max_length=100, blank=True)
     server_ip = models.CharField(max_length=100, blank=True)
 
-    # serialized copies of the client and server instance model fields, for debugging/tracking purposes
-    client_instance = models.TextField(default="{}")
-    server_instance = models.TextField(default="{}")
+    # track the instance IDs for convenient querying, as well as serialized copies of
+    # the client and server instance model fields for debugging/tracking purposes
+    client_instance_id = models.UUIDField(null=True, blank=True)
+    client_instance_json = models.TextField(default="{}")
+    server_instance_id = models.UUIDField(null=True, blank=True)
+    server_instance_json = models.TextField(default="{}")
 
     # used to store other data we may need to know about this sync session
     extra_fields = models.TextField(default="{}")
@@ -252,11 +252,11 @@ class SyncSession(models.Model):
 
     @cached_property
     def client_instance_data(self):
-        return json.loads(self.client_instance)
+        return json.loads(self.client_instance_json)
 
     @cached_property
     def server_instance_data(self):
-        return json.loads(self.server_instance)
+        return json.loads(self.server_instance_json)
 
 
 class TransferSession(models.Model):
@@ -280,7 +280,7 @@ class TransferSession(models.Model):
     bytes_sent = models.BigIntegerField(default=0, null=True, blank=True)
     bytes_received = models.BigIntegerField(default=0, null=True, blank=True)
 
-    sync_session = models.ForeignKey(SyncSession)
+    sync_session = models.ForeignKey(SyncSession, on_delete=models.CASCADE)
 
     # track when the transfer session started and the last time there was activity on it
     start_timestamp = models.DateTimeField(default=timezone.now)
@@ -351,10 +351,10 @@ class TransferSession(models.Model):
 
     def get_touched_record_ids_for_model(self, model):
         if isinstance(model, SyncableModel) or (
-            isinstance(model, six.class_types) and issubclass(model, SyncableModel)
+            isinstance(model, type) and issubclass(model, SyncableModel)
         ):
             model = model.morango_model_name
-        _assert(isinstance(model, six.string_types), "Model must resolve to string")
+        _assert(isinstance(model, str), "Model must resolve to string")
         return Store.objects.filter(
             model_name=model, last_transfer_session_id=self.id
         ).values_list("id", flat=True)
@@ -418,7 +418,7 @@ class StoreQueryset(models.QuerySet):
             self.annotate(id_cast=Cast("id", TextField()))
             # remove dashes from char uuid
             .annotate(
-                fixed_id=Func(F("id_cast"), Value("-"), Value(""), function="replace")
+                fixed_id=Func(F("id_cast"), Value("-"), Value(""), function="replace", output_field=TextField())
             )
             # return as list
             .values_list("fixed_id", flat=True)
@@ -449,7 +449,8 @@ class Store(AbstractStore):
 
     class Meta:
         indexes = [
-            models.Index(fields=['partition'], name='idx_morango_store_partition'),
+            models.Index(fields=["partition"], name="idx_morango_store_partition"),
+            models.Index(fields=["profile", "model_name", "partition", "dirty_bit"], condition=models.Q(dirty_bit=True), name="idx_morango_deserialize"),
         ]
 
     def _deserialize_store_model(self, fk_cache, defer_fks=False):  # noqa: C901
@@ -465,14 +466,14 @@ class Store(AbstractStore):
         klass_model = syncable_models.get_model(self.profile, self.model_name)
         # if store model marked as deleted, attempt to delete in app layer
         if self.deleted:
-            # if hard deleted, propagate to related models
-            if self.hard_deleted:
-                try:
-                    klass_model.objects.get(id=self.id).delete(hard_delete=True)
-                except klass_model.DoesNotExist:
-                    pass
-            else:
-                klass_model.objects.filter(id=self.id).delete()
+            # Don't differentiate between deletion and hard deletion here,
+            # as we don't want to add additional tracking for models in either case,
+            # just to actually delete them.
+            # Import here to avoid circular import, as the utils module
+            # imports core models.
+            from morango.sync.utils import mute_signals
+            with mute_signals(signals.post_delete):
+                klass_model.syncing_objects.filter(id=self.id).delete()
             return None, deferred_fks
         else:
             # load model into memory
@@ -527,7 +528,7 @@ class Buffer(AbstractStore):
     dequeuing into the local store.
     """
 
-    transfer_session = models.ForeignKey(TransferSession)
+    transfer_session = models.ForeignKey(TransferSession, on_delete=models.CASCADE)
     model_uuid = UUIDField()
 
     class Meta:
@@ -608,7 +609,7 @@ class DatabaseMaxCounter(AbstractCounter):
                         )
         else:
             updated_fsic = {}
-            for key, value in six.iteritems(fsics):
+            for key, value in fsics.items():
                 if key in internal_fsic:
                     # if same instance id, update fsic with larger value
                     if fsics[key] > internal_fsic[key]:
@@ -618,7 +619,7 @@ class DatabaseMaxCounter(AbstractCounter):
                     updated_fsic[key] = fsics[key]
 
             # load database max counters
-            for (key, value) in six.iteritems(updated_fsic):
+            for (key, value) in updated_fsic.items():
                 for f in sync_filter:
                     DatabaseMaxCounter.objects.update_or_create(
                         instance_id=key, partition=f, defaults={"counter": value}
@@ -761,7 +762,7 @@ class RecordMaxCounter(AbstractCounter):
     during the sync process.
     """
 
-    store_model = models.ForeignKey(Store)
+    store_model = models.ForeignKey(Store, on_delete=models.CASCADE)
 
     class Meta:
         unique_together = ("store_model", "instance_id")
@@ -773,11 +774,13 @@ class RecordMaxCounterBuffer(AbstractCounter):
     until they are sent or received by another morango instance.
     """
 
-    transfer_session = models.ForeignKey(TransferSession)
+    transfer_session = models.ForeignKey(TransferSession, on_delete=models.CASCADE)
     model_uuid = UUIDField(db_index=True)
 
 
-ForeignKeyReference = namedtuple("ForeignKeyReference", ["from_field", "from_pk", "to_pk"])
+ForeignKeyReference = namedtuple(
+    "ForeignKeyReference", ["from_field", "from_pk", "to_pk"]
+)
 
 
 class SyncableModel(UUIDModelMixin):
@@ -803,6 +806,11 @@ class SyncableModel(UUIDModelMixin):
 
     objects = SyncableModelManager()
 
+    # Add a special syncing_objects queryset to every SyncableModel for use in syncing operations.
+    # This means that we still deal with the entire set of objects when syncing, even if the default
+    # model manager has been overridden to filter the queryset.
+    syncing_objects = SyncableModelManager()
+
     class Meta:
         abstract = True
 
@@ -839,10 +847,8 @@ class SyncableModel(UUIDModelMixin):
         with transaction.atomic():
             if hard_delete:
                 # set hard deletion for all related models
-                for model, instances in six.iteritems(collector.data):
-                    if issubclass(model, SyncableModel) or issubclass(
-                        model, MorangoMPTTModel
-                    ):
+                for model, instances in collector.data.items():
+                    if issubclass(model, SyncableModel):
                         for obj in instances:
                             obj._update_hard_deleted_models()
             return collector.delete()
@@ -908,7 +914,7 @@ class SyncableModel(UUIDModelMixin):
                 ForeignKeyReference(
                     from_field=field.attname,
                     from_pk=self.pk,
-                    to_pk=getattr(self, field.attname)
+                    to_pk=getattr(self, field.attname),
                 )
             )
 
@@ -926,15 +932,8 @@ class SyncableModel(UUIDModelMixin):
                 continue
             if f.attname in self._morango_internal_fields_not_to_serialize:
                 continue
-            # case if model is morango mptt
-            if f.attname in getattr(
-                self,
-                "_internal_mptt_fields_not_to_serialize",
-                "_internal_fields_not_to_serialize",
-            ):
-                continue
-            if hasattr(f, "value_from_object_json_compatible"):
-                data[f.attname] = f.value_from_object_json_compatible(self)
+            if getattr(f, "morango_serialize_to_string", False):
+                data[f.attname] = f.value_to_string(self)
             else:
                 data[f.attname] = f.value_from_object(self)
         return data
@@ -945,7 +944,10 @@ class SyncableModel(UUIDModelMixin):
         kwargs = {}
         for f in cls._meta.concrete_fields:
             if f.attname in dict_model:
-                kwargs[f.attname] = dict_model[f.attname]
+                if getattr(f, "morango_serialize_to_string", False):
+                    kwargs[f.attname] = f.to_python(dict_model[f.attname])
+                else:
+                    kwargs[f.attname] = dict_model[f.attname]
         return cls(**kwargs)
 
     @classmethod

morango/models/fields/crypto.py

@@ -21,6 +21,16 @@ except ImportError:
     M2CRYPTO_EXISTS = False
 
 try:
+    # Pre-empt the PanicException that importing cryptography can cause
+    # when we are using a non-compatible version of cffi on Python 3.13
+    # this happens because of static depdendency bundling in Kolibri
+    import cffi
+
+    if sys.version_info > (3, 13):
+        if hasattr(cffi, "__version_info__"):
+            if cffi.__version_info__ < (1, 17, 1):
+                raise ImportError
+
     from cryptography.hazmat.backends import default_backend
     from cryptography import exceptions as crypto_exceptions
 
@@ -40,11 +50,15 @@ try:
     CRYPTOGRAPHY_EXISTS = True
 except ImportError:
     CRYPTOGRAPHY_EXISTS = False
+except BaseException as e:
+    # Still catch PanicExceptions just in case.
+    if "Python API call failed" in str(e):
+        CRYPTOGRAPHY_EXISTS = False
+    else:
+        # Otherwise raise the error again to avoid silently catching other errors
+        raise
 
-if sys.version_info[0] < 3:
-    from base64 import encodestring as b64encode, decodestring as b64decode
-else:
-    from base64 import encodebytes as b64encode, decodebytes as b64decode
+from base64 import encodebytes as b64encode, decodebytes as b64decode
 
 
 PKCS8_HEADER = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A"
@@ -347,7 +361,7 @@ class RSAKeyBaseField(models.TextField):
 
 
 class PublicKeyField(RSAKeyBaseField):
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         if not value:
             return None
         return Key(public_key_string=value)
@@ -366,7 +380,7 @@ class PublicKeyField(RSAKeyBaseField):
 
 
 class PrivateKeyField(RSAKeyBaseField):
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         if not value:
             return None
         return Key(private_key_string=value)

morango/models/fields/uuids.py

@@ -2,6 +2,7 @@ import hashlib
 import uuid
 
 from django.db import models
+
 from morango.utils import _assert
 
 
@@ -41,7 +42,7 @@ class UUIDField(models.CharField):
                 raise TypeError(self.error_messages["invalid"] % {"value": value})
         return value.hex
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         return self.to_python(value)
 
     def to_python(self, value):

morango/models/utils.py

@@ -1,5 +1,4 @@
 import hashlib
-import ifcfg
 import os
 import platform
 import socket
@@ -7,10 +6,10 @@ import subprocess
 import sys
 import uuid
 
-from .fields.uuids import sha2_uuid
-
+import ifcfg
 from django.conf import settings
-from django.utils import six
+
+from .fields.uuids import sha2_uuid
 
 
 def _get_database_path():
@@ -109,7 +108,7 @@ def _get_android_uuid():
 def _do_salted_hash(value):
     if not value:
         return ""
-    if not isinstance(value, six.string_types):
+    if not isinstance(value, str):
         value = str(value)
     try:
         value = value.encode()
@@ -185,7 +184,7 @@ def _get_mac_address_flags(mac):
     """
     See: https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local
     """
-    if isinstance(mac, six.integer_types):
+    if isinstance(mac, int):
         mac = _mac_int_to_ether(mac)
 
     first_octet = int(mac[:2], base=16)

morango/proquint.py

@@ -4,7 +4,6 @@ The simplest ways to use this module are the :func:`humanize` and :func:`uuid`
 functions. For tighter control over the output, see :class:`HumanHasher`.
 """
 import uuid
-from django.utils import six
 
 # Copyright (c) 2014 SUNET. All rights reserved.
 #
@@ -59,7 +58,7 @@ def from_int(data):
     :type data: int
     :rtype: string
     """
-    if not isinstance(data, six.integer_types):
+    if not isinstance(data, int):
         raise TypeError("Input must be integer")
 
     res = []
@@ -84,7 +83,7 @@ def to_int(data):
     :type data: string
     :rtype: int
     """
-    if not isinstance(data, six.string_types):
+    if not isinstance(data, str):
         raise TypeError("Input must be string")
 
     res = 0

morango/registry.py

@@ -2,12 +2,11 @@
 `SyncableModelRegistry` holds all syncable models for a project, on a per profile basis.
 This class is registered at app load time for morango in `apps.py`.
 """
-import sys
 import inspect
+import sys
 from collections import OrderedDict
 
 from django.db.models.fields.related import ForeignKey
-from django.utils import six
 
 from morango.constants import transfer_stages
 from morango.errors import InvalidMorangoModelConfiguration
@@ -19,7 +18,7 @@ from morango.utils import SETTINGS
 
 def _get_foreign_key_classes(m):
     return set(
-        [field.rel.to for field in m._meta.fields if isinstance(field, ForeignKey)]
+        [field.related_model for field in m._meta.fields if isinstance(field, ForeignKey)]
     )
 
 
@@ -36,6 +35,24 @@ def _multiple_self_ref_fk_check(class_model):
     return False
 
 
+def _check_manager(name, objects):
+    from morango.models.manager import SyncableModelManager
+    from morango.models.query import SyncableModelQuerySet
+    # syncable model checks
+    if not isinstance(objects, SyncableModelManager):
+        raise InvalidMorangoModelConfiguration(
+            "Manager for {} must inherit from SyncableModelManager.".format(
+                name
+            )
+        )
+    if not isinstance(objects.none(), SyncableModelQuerySet):
+        raise InvalidMorangoModelConfiguration(
+            "Queryset for {} model must inherit from SyncableModelQuerySet.".format(
+                name
+            )
+        )
+
+
 class SyncableModelRegistry(object):
     def __init__(self):
         self.profile_models = {}
@@ -99,8 +116,6 @@ class SyncableModelRegistry(object):
 
         import django.apps
         from morango.models.core import SyncableModel
-        from morango.models.manager import SyncableModelManager
-        from morango.models.query import SyncableModelQuerySet
 
         model_list = []
         for model in django.apps.apps.get_models():
@@ -111,49 +126,10 @@ class SyncableModelRegistry(object):
                     raise InvalidMorangoModelConfiguration(
                         "Syncing models with more than 1 self referential ForeignKey is not supported."
                     )
-                try:
-                    from mptt import models
-                    from morango.models.morango_mptt import (
-                        MorangoMPTTModel,
-                        MorangoMPTTTreeManager,
-                        MorangoTreeQuerySet,
-                    )
+                # Check both the objects and the syncing_objects querysets.
+                _check_manager(name, model.objects)
+                _check_manager(name, model.syncing_objects)
 
-                    # mptt syncable model checks
-                    if issubclass(model, models.MPTTModel):
-                        if not issubclass(model, MorangoMPTTModel):
-                            raise InvalidMorangoModelConfiguration(
-                                "{} that inherits from MPTTModel, should instead inherit from MorangoMPTTModel.".format(
-                                    name
-                                )
-                            )
-                        if not isinstance(model.objects, MorangoMPTTTreeManager):
-                            raise InvalidMorangoModelConfiguration(
-                                "Manager for {} must inherit from MorangoMPTTTreeManager.".format(
-                                    name
-                                )
-                            )
-                        if not isinstance(model.objects.none(), MorangoTreeQuerySet):
-                            raise InvalidMorangoModelConfiguration(
-                                "Queryset for {} model must inherit from MorangoTreeQuerySet.".format(
-                                    name
-                                )
-                            )
-                except ImportError:
-                    pass
-                # syncable model checks
-                if not isinstance(model.objects, SyncableModelManager):
-                    raise InvalidMorangoModelConfiguration(
-                        "Manager for {} must inherit from SyncableModelManager.".format(
-                            name
-                        )
-                    )
-                if not isinstance(model.objects.none(), SyncableModelQuerySet):
-                    raise InvalidMorangoModelConfiguration(
-                        "Queryset for {} model must inherit from SyncableModelQuerySet.".format(
-                            name
-                        )
-                    )
                 if model._meta.many_to_many:
                     raise UnsupportedFieldType(
                         "{} model with a ManyToManyField is not supported in morango."
@@ -178,7 +154,7 @@ class SyncableModelRegistry(object):
                     self._insert_model_in_dependency_order(model, profile)
 
         # for each profile, create a dict mapping from morango_model_name to model class
-        for profile, model_list in six.iteritems(self.profile_models):
+        for profile, model_list in self.profile_models.items():
             mapping = OrderedDict()
             for model in model_list:
                 mapping[model.morango_model_name] = model
@@ -196,6 +172,7 @@ class SessionMiddlewareOperations(list):
     Iterable list class that holds and initializes a list of transfer operations as configured
     through Morango settings, and associate the group with a transfer stage by `related_stage`
     """
+
     __slots__ = ("related_stage",)
 
     def __init__(self, related_stage):
@@ -230,7 +207,9 @@ class SessionMiddlewareOperations(list):
                 return result
         else:
             raise NotImplementedError(
-                "Operation for {} stage has no middleware".format(self.related_stage)
+                "Operation for {} stage has no applicable middleware for context '{}'".format(
+                    self.related_stage, context.__class__.__name__
+                )
             )
 
 

morango/sync/backends/base.py

@@ -1,3 +1,5 @@
+from contextlib import contextmanager
+
 from morango.models.core import Buffer
 from morango.models.core import RecordMaxCounter
 from morango.models.core import RecordMaxCounterBuffer
@@ -10,6 +12,19 @@ class BaseSQLWrapper(object):
     def __init__(self, connection):
         self.connection = connection
 
+    def _is_transaction_isolation_error(self, error):
+        """
+        Determine if an error is related to transaction isolation
+        :param error: An exception
+        :return: A bool whether the error is a transaction isolation error
+        """
+        return False
+
+    @contextmanager
+    def _set_transaction_repeatable_read(self):
+        """Set the current transaction isolation level"""
+        yield
+
     def _create_placeholder_list(self, fields, db_values):
         # number of rows to update
         num_of_rows = len(db_values) // len(fields)
@@ -176,3 +191,22 @@ class BaseSQLWrapper(object):
             name=name, fields=", ".join(field_sqls)
         )
         cursor.execute(sql, fields_params)
+
+    def _lock_all_partitions(self, shared=False):
+        """
+        Execute a lock within the database for all partitions, if the backend supports it. The lock
+        should block until acquired
+
+        :param shared: Whether or not the lock is exclusive or shared
+        """
+        pass
+
+    def _lock_partition(self, partition, shared=False):
+        """
+        Execute a lock within the database for a specific partition, if the database supports it.
+        The lock should block until acquired
+
+        :param partition: The partition prefix string to lock
+        :param shared: Whether or not the lock is exclusive or shared
+        """
+        pass