morango 0.6.11__py2.py3-none-any.whl → 0.8.6__py2.py3-none-any.whl

morango/sync/backends/postgres.py

@@ -1,9 +1,24 @@
+import binascii
+import logging
+from contextlib import contextmanager
+
 from .base import BaseSQLWrapper
 from .utils import get_pk_field
+from morango.errors import MorangoDatabaseError
 from morango.models.core import Buffer
 from morango.models.core import RecordMaxCounter
 from morango.models.core import RecordMaxCounterBuffer
 from morango.models.core import Store
+from morango.utils import SETTINGS
+
+
+# advisory lock integers for locking partitions
+LOCK_ALL_PARTITIONS = 1
+LOCK_PARTITION = 2
+
+SIGNED_MAX_INTEGER = 2147483647
+
+logger = logging.getLogger(__name__)
 
 
 class SQLWrapper(BaseSQLWrapper):
@@ -12,6 +27,60 @@ class SQLWrapper(BaseSQLWrapper):
         "CREATE TEMP TABLE {name} ({fields}) ON COMMIT DROP"
     )
 
+    def _is_transaction_isolation_error(self, error):
+        """
+        Determine if an error is related to transaction isolation
+        :param error: An exception
+        :return: A bool whether the error is a transaction isolation error
+        """
+        from psycopg2.extensions import TransactionRollbackError
+
+        # Django can wrap errors, adding it to the `__cause__` attribute
+        for e in (error, getattr(error, '__cause__', None)):
+            if isinstance(e, TransactionRollbackError):
+                return True
+        return False
+
+    @contextmanager
+    def _set_transaction_repeatable_read(self):
+        """Set the current transaction isolation level"""
+        from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ
+
+        # if we're running tests, we should skip modifying the isolation since the test suites
+        # manage their own connections
+        if SETTINGS.MORANGO_TEST_POSTGRESQL:
+            yield
+        else:
+            # setting the transaction isolation must be either done at the BEGIN statement, or
+            # before any reading/writing operations have taken place, which includes creating
+            # savepoints
+            if self.connection.in_atomic_block:
+                raise MorangoDatabaseError("Unable to set transaction isolation during a transaction")
+
+            for savepoint_id in self.connection.savepoint_ids:
+                if savepoint_id is not None:
+                    raise MorangoDatabaseError("Unable to set transaction isolation when savepoints have been created")
+
+            # ensure the postgres database wrapper loads the isolation level, which requires it to
+            # connection to the database.
+            # @see django.db.backends.postgresql.base.DatabaseWrapper.get_new_connection
+            self.connection.ensure_connection()
+            existing_autocommit = self.connection.autocommit
+            existing_isolation_level = self.connection.isolation_level
+
+            try:
+                self.connection.isolation_level = ISOLATION_LEVEL_REPEATABLE_READ
+                self.connection.connection.set_session(isolation_level=ISOLATION_LEVEL_REPEATABLE_READ, autocommit=False)
+                yield
+            finally:
+                self.connection.isolation_level = existing_isolation_level
+                if existing_isolation_level is None:
+                    self.connection.connection.set_isolation_level(existing_isolation_level)
+                else:
+                    # this method cannot accept None values, as they will be ignored
+                    self.connection.connection.set_session(isolation_level=existing_isolation_level)
+                self.connection.connection.set_session(autocommit=existing_autocommit)
+
     def _prepare_with_values(self, name, fields, db_values):
         placeholder_list = self._create_placeholder_list(fields, db_values)
         # convert this list to a string to be passed into raw sql query
@@ -268,3 +337,63 @@ class SQLWrapper(BaseSQLWrapper):
         )
 
         cursor.execute(insert_remaining_rmcb)
+
+    def _execute_lock(self, key1, key2=None, unlock=False, session=False, shared=False, wait=True):
+        """
+        Creates or destroys an advisory lock within postgres
+        :param key1: An int sent to the PG lock function
+        :param key2: A 2nd int sent to the PG lock function
+        :param unlock: A bool representing whether query should use `unlock`
+        :param session: A bool indicating if this should persist outside of transaction
+        :param shared: A bool indicating if this should be shared, otherwise exclusive
+        :param wait: A bool indicating if it should use a `try` PG function
+        """
+        if not session:
+            if not self.connection.in_atomic_block:
+                raise NotImplementedError("Advisory lock requires transaction")
+            if unlock:
+                raise NotImplementedError("Transaction level locks unlock automatically")
+
+        keys = [key1]
+        if key2 is not None:
+            keys.append(key2)
+
+        query = "SELECT pg{_try}_advisory_{xact_}{lock}{_shared}({keys}) AS lock;".format(
+            _try="" if wait else "_try",
+            xact_="" if session else "xact_",
+            lock="unlock" if unlock else "lock",
+            _shared="_shared" if shared else "",
+            keys=", ".join(["%s"] * len(keys))
+        )
+
+        with self.connection.cursor() as c:
+            c.execute(query, keys)
+
+    def _lock_all_partitions(self, shared=False):
+        """
+        Execute a lock within the database for all partitions, if the database supports it.
+
+        :param shared: Whether the lock is exclusive or shared
+        """
+        self._execute_lock(LOCK_ALL_PARTITIONS, shared=shared)
+
+    def _lock_partition(self, partition, shared=False):
+        """
+        Execute a lock within the database for a specific partition, if the database supports it.
+
+        :param partition: The partition prefix string to lock
+        :param shared: Whether the lock is exclusive or shared
+        """
+        # first we open a shared lock on all partitions, so that we don't interfere with concurrent
+        # locks on all partitions or operations that could attempt to open a lock on all partitions
+        # while we've locked only some partitions
+        self._lock_all_partitions(shared=True)
+
+        # Postgres advisory locks use integers, so we have to convert the partition string into
+        # an integer. To do this we use crc32, which returns an unsigned integer. When using two
+        # keys for advisory locks, the two keys are signed integers, so we have to adjust the crc32
+        # value so that it doesn't exceed the maximum signed integer. Turning the partition str into
+        # a crc32 value could produce the same integer for different partitions, but for the
+        # purposes of locking to manage concurrency, this shouldn't be an issue.
+        partition_int = binascii.crc32(partition.encode("utf-8")) - SIGNED_MAX_INTEGER
+        self._execute_lock(LOCK_PARTITION, key2=partition_int, shared=shared)

morango/sync/backends/utils.py

@@ -1,8 +1,7 @@
 import sqlite3
+from functools import lru_cache
 from importlib import import_module
 
-from django.utils.lru_cache import lru_cache
-
 from morango.errors import MorangoError
 
 
@@ -110,15 +109,15 @@ class TemporaryTable(object):
         """
         fields = []
         params = []
-        with self.connection.schema_editor() as schema_editor:
-            for field in self.fields:
-                # generates the SQL expression for the table column
-                field_sql, field_params = schema_editor.column_sql(
-                    self, field, include_default=True
-                )
-                field_sql_name = self.connection.ops.quote_name(field.column)
-                fields.append("{name} {sql}".format(name=field_sql_name, sql=field_sql))
-                params.extend(field_params)
+        schema_editor = self.connection.schema_editor()
+        for field in self.fields:
+            # generates the SQL expression for the table column
+            field_sql, field_params = schema_editor.column_sql(
+                self, field, include_default=True
+            )
+            field_sql_name = self.connection.ops.quote_name(field.column)
+            fields.append("{name} {sql}".format(name=field_sql_name, sql=field_sql))
+            params.extend(field_params)
         with self.connection.cursor() as c:
             self.backend._create_temporary_table(c, self.sql_name, fields, params)
 

morango/sync/context.py

@@ -4,8 +4,8 @@ from morango.errors import MorangoContextUpdateError
 from morango.models.certificates import Filter
 from morango.models.core import SyncSession
 from morango.models.core import TransferSession
-from morango.utils import parse_capabilities_from_server_request
 from morango.utils import CAPABILITIES
+from morango.utils import parse_capabilities_from_server_request
 
 
 class SessionContext(object):
@@ -21,6 +21,7 @@ class SessionContext(object):
         "capabilities",
         "error",
     )
+    max_backoff_interval = 5
 
     def __init__(
         self,
@@ -55,6 +56,20 @@ class SessionContext(object):
             if transfer_session.filter:
                 self.filter = transfer_session.get_filter()
 
+    def prepare(self):
+        """
+        Perform any processing of the session context prior to passing it to the middleware,
+        and return the context as it should be passed to the middleware
+        """
+        return self
+
+    def join(self, context):
+        """
+        Perform any processing of session context after passing it to the middleware, which will
+        receive the result returned by `prepare` after that has happened
+        """
+        pass
+
     def update(
         self,
         transfer_session=None,
@@ -75,7 +90,7 @@ class SessionContext(object):
         :type capabilities: str[]|None
         :type error: BaseException|None
         """
-        if transfer_session and self.transfer_session:
+        if transfer_session and self.transfer_session and transfer_session.id != self.transfer_session.id:
             raise MorangoContextUpdateError("Transfer session already exists")
         elif (
             transfer_session
@@ -84,14 +99,18 @@ class SessionContext(object):
         ):
             raise MorangoContextUpdateError("Sync session mismatch")
 
-        if sync_filter and self.filter:
-            raise MorangoContextUpdateError("Filter already exists")
+        if sync_filter and self.filter and sync_filter != self.filter:
+            if not self.filter.is_subset_of(sync_filter):
+                raise MorangoContextUpdateError("The existing filter must be a subset of the new filter")
+            if transfer_stages.stage(self.stage) > transfer_stages.stage(transfer_stages.INITIALIZING):
+                raise MorangoContextUpdateError("Cannot update filter after initializing stage")
 
         if is_push is not None and self.is_push is not None:
             raise MorangoContextUpdateError("Push/pull method already exists")
 
         self.transfer_session = transfer_session or self.transfer_session
-        self.filter = sync_filter or self.filter
+        if sync_filter or self.filter:
+            self.filter = Filter.add(self.filter, sync_filter)
         self.is_push = is_push if is_push is not None else self.is_push
         self.capabilities = set(capabilities or self.capabilities) & CAPABILITIES
         self.update_state(stage=stage, stage_status=stage_status)
@@ -186,6 +205,7 @@ class LocalSessionContext(SessionContext):
         "request",
         "is_server",
     )
+    max_backoff_interval = 1
 
     def __init__(self, request=None, **kwargs):
         """
@@ -194,14 +214,22 @@ class LocalSessionContext(SessionContext):
             passed in.
         :type request: django.http.request.HttpRequest
         """
-        capabilities = kwargs.pop("capabilities", [])
-        if request is not None:
-            capabilities = parse_capabilities_from_server_request(request)
-
-        super(LocalSessionContext, self).__init__(capabilities=capabilities, **kwargs)
+        super(LocalSessionContext, self).__init__(**kwargs)
         self.request = request
         self.is_server = request is not None
 
+    @classmethod
+    def from_request(cls, request, **kwargs):
+        """
+        Parse capabilities from request and instantiate the LocalSessionContext
+        :param request: The request object
+        :type request: django.http.request.HttpRequest
+        :param kwargs: Any other keyword args for the constructor
+        :rtype: LocalSessionContext
+        """
+        kwargs.update(capabilities=parse_capabilities_from_server_request(request))
+        return LocalSessionContext(request=request, **kwargs)
+
     @property
     def _has_transfer_session(self):
         """
@@ -288,9 +316,7 @@ class NetworkSessionContext(SessionContext):
         :type connection: NetworkSyncConnection
         """
         self.connection = connection
-        super(NetworkSessionContext, self).__init__(
-            capabilities=self.connection.server_info.get("capabilities", []), **kwargs
-        )
+        super(NetworkSessionContext, self).__init__(**kwargs)
 
         # since this is network context, keep local reference to state vars
         self._stage = transfer_stages.INITIALIZING
@@ -317,3 +343,162 @@ class NetworkSessionContext(SessionContext):
         """
         self._stage = stage or self._stage
         self._stage_status = stage_status or self._stage_status
+
+
+class CompositeSessionContext(SessionContext):
+    """
+    A composite context class that acts as a facade for more than one context, to facilitate
+    "simpler" operation on local and remote contexts simultaneously
+    """
+
+    __slots__ = (
+        "children",
+        "_counter",
+        "_stage",
+        "_stage_status",
+    )
+
+    def __init__(self, contexts, *args, **kwargs):
+        """
+        :param contexts: A list of context objects
+        :param args: Args to pass to the parent constructor
+        :param kwargs: Keyword args to pass to the parent constructor
+        """
+        self.children = contexts
+        self._counter = 0
+        self._stage = transfer_stages.INITIALIZING
+        self._stage_status = transfer_statuses.PENDING
+        super(CompositeSessionContext, self).__init__(*args, **kwargs)
+        self._update_attrs(**kwargs)
+
+    @property
+    def max_backoff_interval(self):
+        """
+        The maximum amount of time to wait between retries
+        :return: A number of seconds
+        """
+        return self.prepare().max_backoff_interval
+
+    @property
+    def stage(self):
+        """
+        The stage of the transfer context
+        :return: A transfer_stages.* constant
+        :rtype: str
+        """
+        return self._stage
+
+    @property
+    def stage_status(self):
+        """
+        The status of the transfer context's stage
+        :return: A transfer_statuses.* constant
+        :rtype: str
+        """
+        return self._stage_status
+
+    def _update_attrs(self, **kwargs):
+        """
+        Updates all contexts by applying key/value arguments as attributes. This avoids using the
+        contexts' update methods because some validation is already handled in this class.
+        """
+        for context in self.children:
+            for attr, value in kwargs.items():
+                set_attr = "filter" if attr == "sync_filter" else attr
+                setattr(context, set_attr, value)
+
+    def prepare(self):
+        """
+        Preparing this context will return the current sub context that needs completion
+        """
+        return self.children[self._counter % len(self.children)]
+
+    def join(self, context):
+        """
+        This updates some context attributes that really only make sense logically for what an
+        operation might change during a sync
+
+        :param context: The context that was returned previously from `prepare`
+        :type context: SessionContext
+        """
+        updates = {}
+        if not self.transfer_session and context.transfer_session:
+            # if the transfer session is being resumed, we'd detect a different stage here,
+            # and thus we reset the counter, so we can be sure to start fresh at that stage
+            # on the next invocation of the middleware
+            if (
+                context.transfer_session.transfer_stage
+                and self._stage
+                and context.transfer_session.transfer_stage != self._stage
+            ):
+                self._counter = 0
+                updates.update(
+                    stage=context.transfer_session.transfer_stage,
+                    stage_status=transfer_statuses.PENDING,
+                )
+            updates.update(transfer_session=context.transfer_session)
+        if self.filter != context.filter:
+            updates.update(sync_filter=context.filter)
+        if self.capabilities != context.capabilities:
+            updates.update(capabilities=context.capabilities)
+        if self.error != context.error:
+            updates.update(error=context.error)
+        if updates:
+            self.update(**updates)
+
+    def update(self, stage=None, stage_status=None, **kwargs):
+        # update ourselves, but exclude stage and stage_status
+        super(CompositeSessionContext, self).update(**kwargs)
+        # update children contexts directly, but exclude stage and stage_status
+        self._update_attrs(**kwargs)
+        # handle state changes after updating children
+        self.update_state(stage=stage, stage_status=stage_status)
+
+    def update_state(self, stage=None, stage_status=None):
+        """
+        Updates the state of the transfer
+        :type stage: transfer_stages.*|None
+        :type stage_status: transfer_statuses.*|None
+        """
+        # parent's update method can pass through None values
+        if stage is None and stage_status is None:
+            return
+
+        # advance the composite's stage when we move forward only
+        if stage is not None and transfer_stages.stage(stage) > transfer_stages.stage(
+            self._stage
+        ):
+            self._stage = stage
+
+        # when finishing a stage without an error, we'll increment the counter by one such that
+        # `prepare` returns the next context to process
+        if stage_status == transfer_statuses.COMPLETED:
+            self._counter += 1
+
+        # when we've completed a loop through all contexts (modulus is zero), we want to bring
+        # all the contexts' states up to date
+        if (
+            self._counter % len(self.children) == 0
+            or stage_status == transfer_statuses.ERRORED
+        ):
+            for context in self.children:
+                context.update_state(stage=stage, stage_status=stage_status)
+            if stage_status is not None:
+                self._stage_status = stage_status
+
+    def __getstate__(self):
+        """Return dict of simplified data for serialization"""
+        return dict(
+            children=self.children,
+            counter=self._counter,
+            stage=self._stage,
+            stage_status=self._stage_status,
+        )
+
+    def __setstate__(self, state):
+        """Re-apply dict state after serialization"""
+        self.children = state.get("children", [])
+        self._counter = state.get("counter", 0)
+        self._stage = state.get("stage", None)
+        self._stage_status = state.get("stage_status", None)
+        self.error = state.get("error", None)

morango/sync/controller.py

@@ -1,10 +1,10 @@
 import logging
+import math
 from time import sleep
 
 from morango.constants import transfer_stages
 from morango.constants import transfer_statuses
 from morango.registry import session_middleware
-
 from morango.sync.operations import _deserialize_from_store
 from morango.sync.operations import _serialize_into_store
 from morango.sync.operations import OperationLogger
@@ -130,10 +130,10 @@ class SessionController(object):
         When invoking the middleware, if the status result is:
             PENDING: The controller will continue to invoke the middleware again when this method
                 is called repeatedly, until the status result changes
-            STARTED: The controller will not invoke any middleware until the the status changes,
+            STARTED: The controller will not invoke any middleware until the status changes,
                 and assumes the "started" operation will update the state itself
             COMPLETED: The controller will proceed to invoke the middleware for the next stage
-            ERRORED: The controller will not invoke any middleware until the the status changes,
+            ERRORED: The controller will not invoke any middleware until the status changes,
                 which would require codified resolution of the error outside of the controller
 
         :param target_stage: transfer_stage.* - The transfer stage to proceed to
@@ -187,7 +187,9 @@ class SessionController(object):
         # should always be a non-False status
         return result
 
-    def proceed_to_and_wait_for(self, target_stage, context=None, max_interval=5):
+    def proceed_to_and_wait_for(
+        self, target_stage, context=None, max_interval=None, callback=None
+    ):
         """
         Same as `proceed_to` but waits for a finished status to be returned by sleeping between
         calls to `proceed_to` if status is not complete
@@ -197,18 +199,29 @@ class SessionController(object):
         :param context: Override controller context, or provide it if missing
         :type context: morango.sync.context.SessionContext|None
         :param max_interval: The max time, in seconds, between repeat calls to `.proceed_to`
+        :param callback: A callable to invoke after every attempt
         :return: transfer_status.* - The status of proceeding to that stage,
             which should be `ERRORED` or `COMPLETE`
         :rtype: str
         """
         result = transfer_statuses.PENDING
         tries = 0
+        context = context or self.context
+        max_interval = max_interval or context.max_backoff_interval
+        # solve for the number of tries at which our sleep time will always be max_interval
+        max_interval_tries = math.ceil(math.log(max_interval / 0.3 + 1) / math.log(2))
+
         while result not in transfer_statuses.FINISHED_STATES:
             if tries > 0:
                 # exponential backoff up to max_interval
-                sleep(min(0.3 * (2 ** tries - 1), max_interval))
+                if tries >= max_interval_tries:
+                    sleep(max_interval)
+                else:
+                    sleep(0.3 * (2 ** tries - 1))
             result = self.proceed_to(target_stage, context=context)
             tries += 1
+            if callable(callback):
+                callback()
         return result
 
     def _invoke_middleware(self, context, middleware):
@@ -224,16 +237,24 @@ class SessionController(object):
         stage = middleware.related_stage
         signal = getattr(self.signals, stage)
         at_stage = context.stage == stage
+        prepared_context = None
 
         try:
             context.update(stage=stage, stage_status=transfer_statuses.PENDING)
 
+            # we'll use the prepared context for passing to the middleware and any signal handlers
+            prepared_context = context.prepare()
+
             # only fire "started" when we first try to invoke the stage
             # NOTE: this means that signals.started is not equivalent to transfer_stage.STARTED
             if not at_stage:
-                signal.started.fire(context=context)
+                signal.started.fire(context=prepared_context)
 
-            result = middleware(context)
+            # invoke the middleware with the prepared context
+            result = middleware(prepared_context)
+
+            # return the prepared context back
+            context.join(prepared_context)
 
             # don't update stage result if context's stage was updated during operation
             if context.stage == stage:
@@ -241,15 +262,16 @@ class SessionController(object):
 
             # fire signals based off middleware invocation result; the progress signal if incomplete
             if result == transfer_statuses.COMPLETED:
-                signal.completed.fire(context=context)
+                signal.completed.fire(context=prepared_context)
             else:
-                signal.in_progress.fire(context=context)
+                signal.in_progress.fire(context=prepared_context)
 
-            return result
+            # context should take precedence over result, and was likely updated
+            return context.stage_status
         except Exception as e:
             # always log the error itself
             logger.error(e)
             context.update(stage_status=transfer_statuses.ERRORED, error=e)
             # fire completed signal, after context update. handlers can use context to detect error
-            signal.completed.fire(context=context)
+            signal.completed.fire(context=prepared_context or context)
             return transfer_statuses.ERRORED