morango 0.6.14__py2.py3-none-any.whl → 0.8.6__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. morango/__init__.py +1 -6
  2. morango/api/serializers.py +3 -0
  3. morango/api/viewsets.py +19 -6
  4. morango/apps.py +1 -2
  5. morango/constants/transfer_stages.py +1 -1
  6. morango/constants/transfer_statuses.py +1 -1
  7. morango/management/commands/cleanupsyncs.py +64 -14
  8. morango/migrations/0001_initial.py +0 -2
  9. morango/migrations/0001_squashed_0024_auto_20240129_1757.py +583 -0
  10. morango/migrations/0002_auto_20170511_0400.py +0 -2
  11. morango/migrations/0002_store_idx_morango_deserialize.py +21 -0
  12. morango/migrations/0003_auto_20170519_0543.py +0 -2
  13. morango/migrations/0004_auto_20170520_2112.py +0 -2
  14. morango/migrations/0005_auto_20170629_2139.py +0 -2
  15. morango/migrations/0006_instanceidmodel_system_id.py +0 -2
  16. morango/migrations/0007_auto_20171018_1615.py +0 -2
  17. morango/migrations/0008_auto_20171114_2217.py +0 -2
  18. morango/migrations/0009_auto_20171205_0252.py +0 -2
  19. morango/migrations/0010_auto_20171206_1615.py +0 -2
  20. morango/migrations/0011_sharedkey.py +0 -2
  21. morango/migrations/0012_auto_20180927_1658.py +0 -2
  22. morango/migrations/0013_auto_20190627_1513.py +0 -2
  23. morango/migrations/0014_syncsession_extra_fields.py +0 -2
  24. morango/migrations/0015_auto_20200508_2104.py +2 -3
  25. morango/migrations/0016_store_deserialization_error.py +2 -3
  26. morango/migrations/0017_store_last_transfer_session_id.py +1 -2
  27. morango/migrations/0018_auto_20210714_2216.py +2 -3
  28. morango/migrations/0019_auto_20220113_1807.py +2 -3
  29. morango/migrations/0020_postgres_fix_nullable.py +0 -2
  30. morango/migrations/0021_store_partition_index_create.py +0 -2
  31. morango/migrations/0022_rename_instance_fields.py +23 -0
  32. morango/migrations/0023_add_instance_id_fields.py +24 -0
  33. morango/migrations/0024_auto_20240129_1757.py +28 -0
  34. morango/models/__init__.py +0 -6
  35. morango/models/certificates.py +137 -28
  36. morango/models/core.py +36 -36
  37. morango/models/fields/crypto.py +20 -6
  38. morango/models/fields/uuids.py +2 -1
  39. morango/models/utils.py +5 -6
  40. morango/proquint.py +2 -3
  41. morango/registry.py +23 -47
  42. morango/sync/backends/utils.py +10 -11
  43. morango/sync/context.py +48 -40
  44. morango/sync/controller.py +10 -1
  45. morango/sync/operations.py +1 -1
  46. morango/sync/session.py +11 -0
  47. morango/sync/syncsession.py +41 -22
  48. morango/urls.py +3 -3
  49. morango/utils.py +2 -3
  50. {morango-0.6.14.dist-info → morango-0.8.6.dist-info}/METADATA +29 -14
  51. morango-0.8.6.dist-info/RECORD +79 -0
  52. {morango-0.6.14.dist-info → morango-0.8.6.dist-info}/WHEEL +1 -1
  53. morango/models/morango_mptt.py +0 -33
  54. morango/settings.py +0 -115
  55. morango/wsgi.py +0 -33
  56. morango-0.6.14.dist-info/RECORD +0 -77
  57. {morango-0.6.14.dist-info → morango-0.8.6.dist-info/licenses}/AUTHORS.md +0 -0
  58. {morango-0.6.14.dist-info → morango-0.8.6.dist-info/licenses}/LICENSE +0 -0
  59. {morango-0.6.14.dist-info → morango-0.8.6.dist-info}/top_level.txt +0 -0
@@ -21,6 +21,16 @@ except ImportError:
21
21
  M2CRYPTO_EXISTS = False
22
22
 
23
23
  try:
24
+ # Pre-empt the PanicException that importing cryptography can cause
25
+ # when we are using a non-compatible version of cffi on Python 3.13
26
+ # this happens because of static depdendency bundling in Kolibri
27
+ import cffi
28
+
29
+ if sys.version_info > (3, 13):
30
+ if hasattr(cffi, "__version_info__"):
31
+ if cffi.__version_info__ < (1, 17, 1):
32
+ raise ImportError
33
+
24
34
  from cryptography.hazmat.backends import default_backend
25
35
  from cryptography import exceptions as crypto_exceptions
26
36
 
@@ -40,11 +50,15 @@ try:
40
50
  CRYPTOGRAPHY_EXISTS = True
41
51
  except ImportError:
42
52
  CRYPTOGRAPHY_EXISTS = False
53
+ except BaseException as e:
54
+ # Still catch PanicExceptions just in case.
55
+ if "Python API call failed" in str(e):
56
+ CRYPTOGRAPHY_EXISTS = False
57
+ else:
58
+ # Otherwise raise the error again to avoid silently catching other errors
59
+ raise
43
60
 
44
- if sys.version_info[0] < 3:
45
- from base64 import encodestring as b64encode, decodestring as b64decode
46
- else:
47
- from base64 import encodebytes as b64encode, decodebytes as b64decode
61
+ from base64 import encodebytes as b64encode, decodebytes as b64decode
48
62
 
49
63
 
50
64
  PKCS8_HEADER = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A"
@@ -347,7 +361,7 @@ class RSAKeyBaseField(models.TextField):
347
361
 
348
362
 
349
363
  class PublicKeyField(RSAKeyBaseField):
350
- def from_db_value(self, value, expression, connection, context):
364
+ def from_db_value(self, value, expression, connection):
351
365
  if not value:
352
366
  return None
353
367
  return Key(public_key_string=value)
@@ -366,7 +380,7 @@ class PublicKeyField(RSAKeyBaseField):
366
380
 
367
381
 
368
382
  class PrivateKeyField(RSAKeyBaseField):
369
- def from_db_value(self, value, expression, connection, context):
383
+ def from_db_value(self, value, expression, connection):
370
384
  if not value:
371
385
  return None
372
386
  return Key(private_key_string=value)
@@ -2,6 +2,7 @@ import hashlib
2
2
  import uuid
3
3
 
4
4
  from django.db import models
5
+
5
6
  from morango.utils import _assert
6
7
 
7
8
 
@@ -41,7 +42,7 @@ class UUIDField(models.CharField):
41
42
  raise TypeError(self.error_messages["invalid"] % {"value": value})
42
43
  return value.hex
43
44
 
44
- def from_db_value(self, value, expression, connection, context):
45
+ def from_db_value(self, value, expression, connection):
45
46
  return self.to_python(value)
46
47
 
47
48
  def to_python(self, value):
morango/models/utils.py CHANGED
@@ -1,5 +1,4 @@
1
1
  import hashlib
2
- import ifcfg
3
2
  import os
4
3
  import platform
5
4
  import socket
@@ -7,10 +6,10 @@ import subprocess
7
6
  import sys
8
7
  import uuid
9
8
 
10
- from .fields.uuids import sha2_uuid
11
-
9
+ import ifcfg
12
10
  from django.conf import settings
13
- from django.utils import six
11
+
12
+ from .fields.uuids import sha2_uuid
14
13
 
15
14
 
16
15
  def _get_database_path():
@@ -109,7 +108,7 @@ def _get_android_uuid():
109
108
  def _do_salted_hash(value):
110
109
  if not value:
111
110
  return ""
112
- if not isinstance(value, six.string_types):
111
+ if not isinstance(value, str):
113
112
  value = str(value)
114
113
  try:
115
114
  value = value.encode()
@@ -185,7 +184,7 @@ def _get_mac_address_flags(mac):
185
184
  """
186
185
  See: https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local
187
186
  """
188
- if isinstance(mac, six.integer_types):
187
+ if isinstance(mac, int):
189
188
  mac = _mac_int_to_ether(mac)
190
189
 
191
190
  first_octet = int(mac[:2], base=16)
morango/proquint.py CHANGED
@@ -4,7 +4,6 @@ The simplest ways to use this module are the :func:`humanize` and :func:`uuid`
4
4
  functions. For tighter control over the output, see :class:`HumanHasher`.
5
5
  """
6
6
  import uuid
7
- from django.utils import six
8
7
 
9
8
  # Copyright (c) 2014 SUNET. All rights reserved.
10
9
  #
@@ -59,7 +58,7 @@ def from_int(data):
59
58
  :type data: int
60
59
  :rtype: string
61
60
  """
62
- if not isinstance(data, six.integer_types):
61
+ if not isinstance(data, int):
63
62
  raise TypeError("Input must be integer")
64
63
 
65
64
  res = []
@@ -84,7 +83,7 @@ def to_int(data):
84
83
  :type data: string
85
84
  :rtype: int
86
85
  """
87
- if not isinstance(data, six.string_types):
86
+ if not isinstance(data, str):
88
87
  raise TypeError("Input must be string")
89
88
 
90
89
  res = 0
morango/registry.py CHANGED
@@ -7,7 +7,6 @@ import sys
7
7
  from collections import OrderedDict
8
8
 
9
9
  from django.db.models.fields.related import ForeignKey
10
- from django.utils import six
11
10
 
12
11
  from morango.constants import transfer_stages
13
12
  from morango.errors import InvalidMorangoModelConfiguration
@@ -19,7 +18,7 @@ from morango.utils import SETTINGS
19
18
 
20
19
  def _get_foreign_key_classes(m):
21
20
  return set(
22
- [field.rel.to for field in m._meta.fields if isinstance(field, ForeignKey)]
21
+ [field.related_model for field in m._meta.fields if isinstance(field, ForeignKey)]
23
22
  )
24
23
 
25
24
 
@@ -36,6 +35,24 @@ def _multiple_self_ref_fk_check(class_model):
36
35
  return False
37
36
 
38
37
 
38
+ def _check_manager(name, objects):
39
+ from morango.models.manager import SyncableModelManager
40
+ from morango.models.query import SyncableModelQuerySet
41
+ # syncable model checks
42
+ if not isinstance(objects, SyncableModelManager):
43
+ raise InvalidMorangoModelConfiguration(
44
+ "Manager for {} must inherit from SyncableModelManager.".format(
45
+ name
46
+ )
47
+ )
48
+ if not isinstance(objects.none(), SyncableModelQuerySet):
49
+ raise InvalidMorangoModelConfiguration(
50
+ "Queryset for {} model must inherit from SyncableModelQuerySet.".format(
51
+ name
52
+ )
53
+ )
54
+
55
+
39
56
  class SyncableModelRegistry(object):
40
57
  def __init__(self):
41
58
  self.profile_models = {}
@@ -99,8 +116,6 @@ class SyncableModelRegistry(object):
99
116
 
100
117
  import django.apps
101
118
  from morango.models.core import SyncableModel
102
- from morango.models.manager import SyncableModelManager
103
- from morango.models.query import SyncableModelQuerySet
104
119
 
105
120
  model_list = []
106
121
  for model in django.apps.apps.get_models():
@@ -111,49 +126,10 @@ class SyncableModelRegistry(object):
111
126
  raise InvalidMorangoModelConfiguration(
112
127
  "Syncing models with more than 1 self referential ForeignKey is not supported."
113
128
  )
114
- try:
115
- from mptt import models
116
- from morango.models.morango_mptt import (
117
- MorangoMPTTModel,
118
- MorangoMPTTTreeManager,
119
- MorangoTreeQuerySet,
120
- )
129
+ # Check both the objects and the syncing_objects querysets.
130
+ _check_manager(name, model.objects)
131
+ _check_manager(name, model.syncing_objects)
121
132
 
122
- # mptt syncable model checks
123
- if issubclass(model, models.MPTTModel):
124
- if not issubclass(model, MorangoMPTTModel):
125
- raise InvalidMorangoModelConfiguration(
126
- "{} that inherits from MPTTModel, should instead inherit from MorangoMPTTModel.".format(
127
- name
128
- )
129
- )
130
- if not isinstance(model.objects, MorangoMPTTTreeManager):
131
- raise InvalidMorangoModelConfiguration(
132
- "Manager for {} must inherit from MorangoMPTTTreeManager.".format(
133
- name
134
- )
135
- )
136
- if not isinstance(model.objects.none(), MorangoTreeQuerySet):
137
- raise InvalidMorangoModelConfiguration(
138
- "Queryset for {} model must inherit from MorangoTreeQuerySet.".format(
139
- name
140
- )
141
- )
142
- except ImportError:
143
- pass
144
- # syncable model checks
145
- if not isinstance(model.objects, SyncableModelManager):
146
- raise InvalidMorangoModelConfiguration(
147
- "Manager for {} must inherit from SyncableModelManager.".format(
148
- name
149
- )
150
- )
151
- if not isinstance(model.objects.none(), SyncableModelQuerySet):
152
- raise InvalidMorangoModelConfiguration(
153
- "Queryset for {} model must inherit from SyncableModelQuerySet.".format(
154
- name
155
- )
156
- )
157
133
  if model._meta.many_to_many:
158
134
  raise UnsupportedFieldType(
159
135
  "{} model with a ManyToManyField is not supported in morango."
@@ -178,7 +154,7 @@ class SyncableModelRegistry(object):
178
154
  self._insert_model_in_dependency_order(model, profile)
179
155
 
180
156
  # for each profile, create a dict mapping from morango_model_name to model class
181
- for profile, model_list in six.iteritems(self.profile_models):
157
+ for profile, model_list in self.profile_models.items():
182
158
  mapping = OrderedDict()
183
159
  for model in model_list:
184
160
  mapping[model.morango_model_name] = model
@@ -1,8 +1,7 @@
1
1
  import sqlite3
2
+ from functools import lru_cache
2
3
  from importlib import import_module
3
4
 
4
- from django.utils.lru_cache import lru_cache
5
-
6
5
  from morango.errors import MorangoError
7
6
 
8
7
 
@@ -110,15 +109,15 @@ class TemporaryTable(object):
110
109
  """
111
110
  fields = []
112
111
  params = []
113
- with self.connection.schema_editor() as schema_editor:
114
- for field in self.fields:
115
- # generates the SQL expression for the table column
116
- field_sql, field_params = schema_editor.column_sql(
117
- self, field, include_default=True
118
- )
119
- field_sql_name = self.connection.ops.quote_name(field.column)
120
- fields.append("{name} {sql}".format(name=field_sql_name, sql=field_sql))
121
- params.extend(field_params)
112
+ schema_editor = self.connection.schema_editor()
113
+ for field in self.fields:
114
+ # generates the SQL expression for the table column
115
+ field_sql, field_params = schema_editor.column_sql(
116
+ self, field, include_default=True
117
+ )
118
+ field_sql_name = self.connection.ops.quote_name(field.column)
119
+ fields.append("{name} {sql}".format(name=field_sql_name, sql=field_sql))
120
+ params.extend(field_params)
122
121
  with self.connection.cursor() as c:
123
122
  self.backend._create_temporary_table(c, self.sql_name, fields, params)
124
123
 
morango/sync/context.py CHANGED
@@ -63,6 +63,13 @@ class SessionContext(object):
63
63
  """
64
64
  return self
65
65
 
66
+ def join(self, context):
67
+ """
68
+ Perform any processing of session context after passing it to the middleware, which will
69
+ receive the result returned by `prepare` after that has happened
70
+ """
71
+ pass
72
+
66
73
  def update(
67
74
  self,
68
75
  transfer_session=None,
@@ -83,7 +90,7 @@ class SessionContext(object):
83
90
  :type capabilities: str[]|None
84
91
  :type error: BaseException|None
85
92
  """
86
- if transfer_session and self.transfer_session:
93
+ if transfer_session and self.transfer_session and transfer_session.id != self.transfer_session.id:
87
94
  raise MorangoContextUpdateError("Transfer session already exists")
88
95
  elif (
89
96
  transfer_session
@@ -92,14 +99,18 @@ class SessionContext(object):
92
99
  ):
93
100
  raise MorangoContextUpdateError("Sync session mismatch")
94
101
 
95
- if sync_filter and self.filter:
96
- raise MorangoContextUpdateError("Filter already exists")
102
+ if sync_filter and self.filter and sync_filter != self.filter:
103
+ if not self.filter.is_subset_of(sync_filter):
104
+ raise MorangoContextUpdateError("The existing filter must be a subset of the new filter")
105
+ if transfer_stages.stage(self.stage) > transfer_stages.stage(transfer_stages.INITIALIZING):
106
+ raise MorangoContextUpdateError("Cannot update filter after initializing stage")
97
107
 
98
108
  if is_push is not None and self.is_push is not None:
99
109
  raise MorangoContextUpdateError("Push/pull method already exists")
100
110
 
101
111
  self.transfer_session = transfer_session or self.transfer_session
102
- self.filter = sync_filter or self.filter
112
+ if sync_filter or self.filter:
113
+ self.filter = Filter.add(self.filter, sync_filter)
103
114
  self.is_push = is_push if is_push is not None else self.is_push
104
115
  self.capabilities = set(capabilities or self.capabilities) & CAPABILITIES
105
116
  self.update_state(stage=stage, stage_status=stage_status)
@@ -402,13 +413,40 @@ class CompositeSessionContext(SessionContext):
402
413
  """
403
414
  return self.children[self._counter % len(self.children)]
404
415
 
416
+ def join(self, context):
417
+ """
418
+ This updates some context attributes that really only make sense logically for what an
419
+ operation might change during a sync
420
+
421
+ :param context: The context that was returned previously from `prepare`
422
+ :type context: SessionContext
423
+ """
424
+ updates = {}
425
+ if not self.transfer_session and context.transfer_session:
426
+ # if the transfer session is being resumed, we'd detect a different stage here,
427
+ # and thus we reset the counter, so we can be sure to start fresh at that stage
428
+ # on the next invocation of the middleware
429
+ if (
430
+ context.transfer_session.transfer_stage
431
+ and self._stage
432
+ and context.transfer_session.transfer_stage != self._stage
433
+ ):
434
+ self._counter = 0
435
+ updates.update(
436
+ stage=context.transfer_session.transfer_stage,
437
+ stage_status=transfer_statuses.PENDING,
438
+ )
439
+ updates.update(transfer_session=context.transfer_session)
440
+ if self.filter != context.filter:
441
+ updates.update(sync_filter=context.filter)
442
+ if self.capabilities != context.capabilities:
443
+ updates.update(capabilities=context.capabilities)
444
+ if self.error != context.error:
445
+ updates.update(error=context.error)
446
+ if updates:
447
+ self.update(**updates)
448
+
405
449
  def update(self, stage=None, stage_status=None, **kwargs):
406
- """
407
- Updates the context object and its state
408
- :param stage: The str transfer stage
409
- :param stage_status: The str transfer stage status
410
- :param kwargs: Other arguments to update the context with
411
- """
412
450
  # update ourselves, but exclude stage and stage_status
413
451
  super(CompositeSessionContext, self).update(**kwargs)
414
452
  # update children contexts directly, but exclude stage and stage_status
@@ -416,36 +454,6 @@ class CompositeSessionContext(SessionContext):
416
454
  # handle state changes after updating children
417
455
  self.update_state(stage=stage, stage_status=stage_status)
418
456
 
419
- # During the initializing stage, we want to make sure to synchronize the transfer session
420
- # object between the composite and children contexts, using whatever child context's
421
- # transfer session object that was updated on the context during initialization
422
- current_stage = stage or self._stage
423
- if not self.transfer_session and current_stage == transfer_stages.INITIALIZING:
424
- try:
425
- transfer_session = next(
426
- c.transfer_session for c in self.children if c.transfer_session
427
- )
428
- # prepare an updates dictionary, so we can update everything at once
429
- updates = dict(transfer_session=transfer_session)
430
-
431
- # if the transfer session is being resumed, we'd detect a different stage here,
432
- # and thus we reset the counter, so we can be sure to start fresh at that stage
433
- # on the next invocation of the middleware
434
- if (
435
- transfer_session.transfer_stage
436
- and transfer_session.transfer_stage != current_stage
437
- ):
438
- self._counter = 0
439
- updates.update(
440
- stage=transfer_session.transfer_stage,
441
- stage_status=transfer_statuses.PENDING,
442
- )
443
-
444
- # recurse into update with transfer session and possibly state updates too
445
- self.update(**updates)
446
- except StopIteration:
447
- pass
448
-
449
457
  def update_state(self, stage=None, stage_status=None):
450
458
  """
451
459
  Updates the state of the transfer
@@ -1,4 +1,5 @@
1
1
  import logging
2
+ import math
2
3
  from time import sleep
3
4
 
4
5
  from morango.constants import transfer_stages
@@ -207,11 +208,16 @@ class SessionController(object):
207
208
  tries = 0
208
209
  context = context or self.context
209
210
  max_interval = max_interval or context.max_backoff_interval
211
+ # solve for the number of tries at which our sleep time will always be max_interval
212
+ max_interval_tries = math.ceil(math.log(max_interval / 0.3 + 1) / math.log(2))
210
213
 
211
214
  while result not in transfer_statuses.FINISHED_STATES:
212
215
  if tries > 0:
213
216
  # exponential backoff up to max_interval
214
- sleep(min(0.3 * (2 ** tries - 1), max_interval))
217
+ if tries >= max_interval_tries:
218
+ sleep(max_interval)
219
+ else:
220
+ sleep(0.3 * (2 ** tries - 1))
215
221
  result = self.proceed_to(target_stage, context=context)
216
222
  tries += 1
217
223
  if callable(callback):
@@ -247,6 +253,9 @@ class SessionController(object):
247
253
  # invoke the middleware with the prepared context
248
254
  result = middleware(prepared_context)
249
255
 
256
+ # return the prepared context back
257
+ context.join(prepared_context)
258
+
250
259
  # don't update stage result if context's stage was updated during operation
251
260
  if context.stage == stage:
252
261
  context.update(stage_status=result)
@@ -145,7 +145,7 @@ def _serialize_into_store(profile, filter=None):
145
145
  for model in syncable_models.get_models(profile):
146
146
  new_store_records = []
147
147
  new_rmc_records = []
148
- klass_queryset = model.objects.filter(_morango_dirty_bit=True)
148
+ klass_queryset = model.syncing_objects.filter(_morango_dirty_bit=True)
149
149
  if prefix_condition:
150
150
  klass_queryset = klass_queryset.filter(prefix_condition)
151
151
  store_records_dict = Store.objects.in_bulk(
morango/sync/session.py CHANGED
@@ -1,11 +1,13 @@
1
1
  import logging
2
2
 
3
3
  from requests import exceptions
4
+ from morango import __version__
4
5
  from requests.sessions import Session
5
6
  from requests.utils import super_len
6
7
  from requests.packages.urllib3.util.url import parse_url
7
8
 
8
9
  from morango.utils import serialize_capabilities_to_client_request
10
+ from morango.utils import SETTINGS
9
11
 
10
12
 
11
13
  logger = logging.getLogger(__name__)
@@ -35,6 +37,15 @@ class SessionWrapper(Session):
35
37
  bytes_sent = 0
36
38
  bytes_received = 0
37
39
 
40
+ def __init__(self):
41
+ super(SessionWrapper, self).__init__()
42
+ user_agent_header = "morango/{}".format(__version__)
43
+ if SETTINGS.CUSTOM_INSTANCE_INFO is not None:
44
+ instances = list(SETTINGS.CUSTOM_INSTANCE_INFO)
45
+ if instances:
46
+ user_agent_header += " " + "{}/{}".format(instances[0], SETTINGS.CUSTOM_INSTANCE_INFO.get(instances[0]))
47
+ self.headers["User-Agent"] = "{} {}".format(user_agent_header, self.headers["User-Agent"])
48
+
38
49
  def request(self, method, url, **kwargs):
39
50
  response = None
40
51
  try:
@@ -7,15 +7,14 @@ import os
7
7
  import socket
8
8
  import uuid
9
9
  from io import BytesIO
10
+ from urllib.parse import urljoin
11
+ from urllib.parse import urlparse
10
12
 
11
13
  from django.utils import timezone
12
- from django.utils.six import iteritems
13
- from django.utils.six import raise_from
14
- from django.utils.six.moves.urllib.parse import urljoin
15
- from django.utils.six.moves.urllib.parse import urlparse
16
14
  from requests.adapters import HTTPAdapter
17
15
  from requests.exceptions import HTTPError
18
16
  from requests.packages.urllib3.util.retry import Retry
17
+ from django.db import transaction, connection
19
18
 
20
19
  from .session import SessionWrapper
21
20
  from morango.api.serializers import CertificateSerializer
@@ -30,9 +29,11 @@ from morango.errors import MorangoError
30
29
  from morango.errors import MorangoResumeSyncError
31
30
  from morango.errors import MorangoServerDoesNotAllowNewCertPush
32
31
  from morango.models.certificates import Certificate
32
+ from morango.models.certificates import Filter
33
33
  from morango.models.certificates import Key
34
34
  from morango.models.core import InstanceIDModel
35
35
  from morango.models.core import SyncSession
36
+ from morango.sync.backends.utils import load_backend
36
37
  from morango.sync.context import CompositeSessionContext
37
38
  from morango.sync.context import LocalSessionContext
38
39
  from morango.sync.context import NetworkSessionContext
@@ -41,6 +42,7 @@ from morango.sync.utils import SyncSignal
41
42
  from morango.sync.utils import SyncSignalGroup
42
43
  from morango.utils import CAPABILITIES
43
44
  from morango.utils import pid_exists
45
+ from morango.sync.utils import lock_partitions
44
46
 
45
47
  if GZIP_BUFFER_POST in CAPABILITIES:
46
48
  from gzip import GzipFile
@@ -48,6 +50,7 @@ if GZIP_BUFFER_POST in CAPABILITIES:
48
50
 
49
51
  logger = logging.getLogger(__name__)
50
52
 
53
+ DBBackend = load_backend(connection)
51
54
 
52
55
  def _join_with_logical_operator(lst, operator):
53
56
  op = ") {operator} (".format(operator=operator)
@@ -225,6 +228,14 @@ class NetworkSyncConnection(Connection):
225
228
  if not server_cert.verify(message, session_resp.json().get("signature")):
226
229
  raise CertificateSignatureInvalid()
227
230
 
231
+ client_instance = InstanceIDModel.get_or_create_current_instance()[0]
232
+
233
+ server_instance_json = session_resp.json().get("server_instance") or "{}"
234
+ server_instance_id = None
235
+ if server_instance_json:
236
+ server_instance = json.loads(server_instance_json)
237
+ server_instance_id = server_instance.get("id")
238
+
228
239
  # build the data to be used for creating our own syncsession
229
240
  data = {
230
241
  "id": data["id"],
@@ -239,23 +250,29 @@ class NetworkSyncConnection(Connection):
239
250
  "connection_path": self.base_url,
240
251
  "client_ip": data["client_ip"],
241
252
  "server_ip": data["server_ip"],
242
- "client_instance": json.dumps(
253
+ "client_instance_id": client_instance.id,
254
+ "client_instance_json": json.dumps(
243
255
  InstanceIDSerializer(
244
- InstanceIDModel.get_or_create_current_instance()[0]
256
+ client_instance
245
257
  ).data
246
258
  ),
247
- "server_instance": session_resp.json().get("server_instance") or "{}",
259
+ "server_instance_id": server_instance_id,
260
+ "server_instance_json": session_resp.json().get("server_instance") or "{}",
248
261
  "process_id": os.getpid(),
249
262
  }
250
263
  sync_session = SyncSession.objects.create(**data)
251
264
  return SyncSessionClient(self, sync_session)
252
265
 
253
- def resume_sync_session(self, sync_session_id, chunk_size=None):
266
+ def resume_sync_session(self, sync_session_id, chunk_size=None, ignore_existing_process=False):
254
267
  """
255
268
  Resumes an existing sync session given an ID
256
269
 
257
270
  :param sync_session_id: The UUID of the `SyncSession` to resume
258
271
  :param chunk_size: An optional parameter specifying the size for each transferred chunk
272
+ :type chunk_size: int
273
+ :param ignore_existing_process:An optional parameter specifying whether to ignore an
274
+ existing active process ID
275
+ :type ignore_existing_process: bool
259
276
  :return: A SyncSessionClient instance
260
277
  :rtype: SyncSessionClient
261
278
  """
@@ -271,7 +288,8 @@ class NetworkSyncConnection(Connection):
271
288
 
272
289
  # check that process of existing session isn't still running
273
290
  if (
274
- sync_session.process_id
291
+ not ignore_existing_process
292
+ and sync_session.process_id
275
293
  and sync_session.process_id != os.getpid()
276
294
  and pid_exists(sync_session.process_id)
277
295
  ):
@@ -284,7 +302,7 @@ class NetworkSyncConnection(Connection):
284
302
  try:
285
303
  self._get_sync_session(sync_session)
286
304
  except HTTPError as e:
287
- raise_from(MorangoResumeSyncError("Failure resuming sync session"), e)
305
+ raise MorangoResumeSyncError("Failure resuming sync session") from e
288
306
 
289
307
  # update process id
290
308
  sync_session.process_id = os.getpid()
@@ -338,11 +356,15 @@ class NetworkSyncConnection(Connection):
338
356
  cert_chain_response = self._get_certificate_chain(
339
357
  params={"ancestors_of": parent_cert.id}
340
358
  )
341
-
342
- # upon receiving cert chain from server, we attempt to save the chain into our records
343
- Certificate.save_certificate_chain(
344
- cert_chain_response.json(), expected_last_id=parent_cert.id
345
- )
359
+ cert_chain = cert_chain_response.json()
360
+ with transaction.atomic():
361
+ lock_partitions(DBBackend, sync_filter=Filter(cert_chain[0]["id"]))
362
+ # check again, now that we have a lock
363
+ if not Certificate.objects.filter(id=parent_cert.id).exists():
364
+ # upon receiving cert chain from server, we attempt to save the chain into our records
365
+ Certificate.save_certificate_chain(
366
+ cert_chain, expected_last_id=parent_cert.id
367
+ )
346
368
 
347
369
  csr_key = Key()
348
370
  # build up data for csr
@@ -425,7 +447,7 @@ class NetworkSyncConnection(Connection):
425
447
  # convert user arguments into query str for passing to auth layer
426
448
  if isinstance(userargs, dict):
427
449
  userargs = "&".join(
428
- ["{}={}".format(key, val) for (key, val) in iteritems(userargs)]
450
+ ["{}={}".format(key, val) for (key, val) in userargs.items()]
429
451
  )
430
452
  return self.session.post(
431
453
  self.urlresolve(api_urls.CERTIFICATE), json=data, auth=(userargs, password)
@@ -612,12 +634,9 @@ class TransferClient(object):
612
634
  """
613
635
  result = self.controller.proceed_to_and_wait_for(stage, callback=callback)
614
636
  if result == transfer_statuses.ERRORED:
615
- raise_from(
616
- MorangoError(
617
- error_msg or "Stage `{}` failed".format(self.context.stage)
618
- ),
619
- self.context.error,
620
- )
637
+ raise MorangoError(
638
+ error_msg or "Stage `{}` failed".format(self.context.stage)
639
+ ) from self.context.error
621
640
 
622
641
  def initialize(self, sync_filter):
623
642
  """
morango/urls.py CHANGED
@@ -1,4 +1,4 @@
1
- from django.conf.urls import include
2
- from django.conf.urls import url
1
+ from django.urls import include
2
+ from django.urls import path
3
3
 
4
- urlpatterns = [url(r"^api/morango/v1/", include("morango.api.urls"))]
4
+ urlpatterns = [path("api/morango/v1/", include("morango.api.urls"))]
morango/utils.py CHANGED
@@ -1,14 +1,13 @@
1
1
  import os
2
- import six
3
2
  from importlib import import_module
4
3
 
5
4
  from django.conf import settings
6
5
 
7
6
  from morango.constants import settings as default_settings
8
7
  from morango.constants.capabilities import ALLOW_CERTIFICATE_PUSHING
9
- from morango.constants.capabilities import GZIP_BUFFER_POST
10
8
  from morango.constants.capabilities import ASYNC_OPERATIONS
11
9
  from morango.constants.capabilities import FSIC_V2_FORMAT
10
+ from morango.constants.capabilities import GZIP_BUFFER_POST
12
11
 
13
12
 
14
13
  def do_import(import_string):
@@ -32,7 +31,7 @@ class Settings(object):
32
31
  def __getattribute__(self, key):
33
32
  """Coalesces settings with the defaults"""
34
33
  value = getattr(settings, key, getattr(default_settings, key, None))
35
- if key == "MORANGO_INSTANCE_INFO" and isinstance(value, six.string_types):
34
+ if key == "MORANGO_INSTANCE_INFO" and isinstance(value, str):
36
35
  value = dict(do_import(value))
37
36
  return value
38
37