morango 0.6.14__py2.py3-none-any.whl → 0.8.6__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- morango/__init__.py +1 -6
- morango/api/serializers.py +3 -0
- morango/api/viewsets.py +19 -6
- morango/apps.py +1 -2
- morango/constants/transfer_stages.py +1 -1
- morango/constants/transfer_statuses.py +1 -1
- morango/management/commands/cleanupsyncs.py +64 -14
- morango/migrations/0001_initial.py +0 -2
- morango/migrations/0001_squashed_0024_auto_20240129_1757.py +583 -0
- morango/migrations/0002_auto_20170511_0400.py +0 -2
- morango/migrations/0002_store_idx_morango_deserialize.py +21 -0
- morango/migrations/0003_auto_20170519_0543.py +0 -2
- morango/migrations/0004_auto_20170520_2112.py +0 -2
- morango/migrations/0005_auto_20170629_2139.py +0 -2
- morango/migrations/0006_instanceidmodel_system_id.py +0 -2
- morango/migrations/0007_auto_20171018_1615.py +0 -2
- morango/migrations/0008_auto_20171114_2217.py +0 -2
- morango/migrations/0009_auto_20171205_0252.py +0 -2
- morango/migrations/0010_auto_20171206_1615.py +0 -2
- morango/migrations/0011_sharedkey.py +0 -2
- morango/migrations/0012_auto_20180927_1658.py +0 -2
- morango/migrations/0013_auto_20190627_1513.py +0 -2
- morango/migrations/0014_syncsession_extra_fields.py +0 -2
- morango/migrations/0015_auto_20200508_2104.py +2 -3
- morango/migrations/0016_store_deserialization_error.py +2 -3
- morango/migrations/0017_store_last_transfer_session_id.py +1 -2
- morango/migrations/0018_auto_20210714_2216.py +2 -3
- morango/migrations/0019_auto_20220113_1807.py +2 -3
- morango/migrations/0020_postgres_fix_nullable.py +0 -2
- morango/migrations/0021_store_partition_index_create.py +0 -2
- morango/migrations/0022_rename_instance_fields.py +23 -0
- morango/migrations/0023_add_instance_id_fields.py +24 -0
- morango/migrations/0024_auto_20240129_1757.py +28 -0
- morango/models/__init__.py +0 -6
- morango/models/certificates.py +137 -28
- morango/models/core.py +36 -36
- morango/models/fields/crypto.py +20 -6
- morango/models/fields/uuids.py +2 -1
- morango/models/utils.py +5 -6
- morango/proquint.py +2 -3
- morango/registry.py +23 -47
- morango/sync/backends/utils.py +10 -11
- morango/sync/context.py +48 -40
- morango/sync/controller.py +10 -1
- morango/sync/operations.py +1 -1
- morango/sync/session.py +11 -0
- morango/sync/syncsession.py +41 -22
- morango/urls.py +3 -3
- morango/utils.py +2 -3
- {morango-0.6.14.dist-info → morango-0.8.6.dist-info}/METADATA +29 -14
- morango-0.8.6.dist-info/RECORD +79 -0
- {morango-0.6.14.dist-info → morango-0.8.6.dist-info}/WHEEL +1 -1
- morango/models/morango_mptt.py +0 -33
- morango/settings.py +0 -115
- morango/wsgi.py +0 -33
- morango-0.6.14.dist-info/RECORD +0 -77
- {morango-0.6.14.dist-info → morango-0.8.6.dist-info/licenses}/AUTHORS.md +0 -0
- {morango-0.6.14.dist-info → morango-0.8.6.dist-info/licenses}/LICENSE +0 -0
- {morango-0.6.14.dist-info → morango-0.8.6.dist-info}/top_level.txt +0 -0
morango/models/fields/crypto.py
CHANGED
|
@@ -21,6 +21,16 @@ except ImportError:
|
|
|
21
21
|
M2CRYPTO_EXISTS = False
|
|
22
22
|
|
|
23
23
|
try:
|
|
24
|
+
# Pre-empt the PanicException that importing cryptography can cause
|
|
25
|
+
# when we are using a non-compatible version of cffi on Python 3.13
|
|
26
|
+
# this happens because of static depdendency bundling in Kolibri
|
|
27
|
+
import cffi
|
|
28
|
+
|
|
29
|
+
if sys.version_info > (3, 13):
|
|
30
|
+
if hasattr(cffi, "__version_info__"):
|
|
31
|
+
if cffi.__version_info__ < (1, 17, 1):
|
|
32
|
+
raise ImportError
|
|
33
|
+
|
|
24
34
|
from cryptography.hazmat.backends import default_backend
|
|
25
35
|
from cryptography import exceptions as crypto_exceptions
|
|
26
36
|
|
|
@@ -40,11 +50,15 @@ try:
|
|
|
40
50
|
CRYPTOGRAPHY_EXISTS = True
|
|
41
51
|
except ImportError:
|
|
42
52
|
CRYPTOGRAPHY_EXISTS = False
|
|
53
|
+
except BaseException as e:
|
|
54
|
+
# Still catch PanicExceptions just in case.
|
|
55
|
+
if "Python API call failed" in str(e):
|
|
56
|
+
CRYPTOGRAPHY_EXISTS = False
|
|
57
|
+
else:
|
|
58
|
+
# Otherwise raise the error again to avoid silently catching other errors
|
|
59
|
+
raise
|
|
43
60
|
|
|
44
|
-
|
|
45
|
-
from base64 import encodestring as b64encode, decodestring as b64decode
|
|
46
|
-
else:
|
|
47
|
-
from base64 import encodebytes as b64encode, decodebytes as b64decode
|
|
61
|
+
from base64 import encodebytes as b64encode, decodebytes as b64decode
|
|
48
62
|
|
|
49
63
|
|
|
50
64
|
PKCS8_HEADER = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A"
|
|
@@ -347,7 +361,7 @@ class RSAKeyBaseField(models.TextField):
|
|
|
347
361
|
|
|
348
362
|
|
|
349
363
|
class PublicKeyField(RSAKeyBaseField):
|
|
350
|
-
def from_db_value(self, value, expression, connection
|
|
364
|
+
def from_db_value(self, value, expression, connection):
|
|
351
365
|
if not value:
|
|
352
366
|
return None
|
|
353
367
|
return Key(public_key_string=value)
|
|
@@ -366,7 +380,7 @@ class PublicKeyField(RSAKeyBaseField):
|
|
|
366
380
|
|
|
367
381
|
|
|
368
382
|
class PrivateKeyField(RSAKeyBaseField):
|
|
369
|
-
def from_db_value(self, value, expression, connection
|
|
383
|
+
def from_db_value(self, value, expression, connection):
|
|
370
384
|
if not value:
|
|
371
385
|
return None
|
|
372
386
|
return Key(private_key_string=value)
|
morango/models/fields/uuids.py
CHANGED
|
@@ -2,6 +2,7 @@ import hashlib
|
|
|
2
2
|
import uuid
|
|
3
3
|
|
|
4
4
|
from django.db import models
|
|
5
|
+
|
|
5
6
|
from morango.utils import _assert
|
|
6
7
|
|
|
7
8
|
|
|
@@ -41,7 +42,7 @@ class UUIDField(models.CharField):
|
|
|
41
42
|
raise TypeError(self.error_messages["invalid"] % {"value": value})
|
|
42
43
|
return value.hex
|
|
43
44
|
|
|
44
|
-
def from_db_value(self, value, expression, connection
|
|
45
|
+
def from_db_value(self, value, expression, connection):
|
|
45
46
|
return self.to_python(value)
|
|
46
47
|
|
|
47
48
|
def to_python(self, value):
|
morango/models/utils.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import hashlib
|
|
2
|
-
import ifcfg
|
|
3
2
|
import os
|
|
4
3
|
import platform
|
|
5
4
|
import socket
|
|
@@ -7,10 +6,10 @@ import subprocess
|
|
|
7
6
|
import sys
|
|
8
7
|
import uuid
|
|
9
8
|
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
import ifcfg
|
|
12
10
|
from django.conf import settings
|
|
13
|
-
|
|
11
|
+
|
|
12
|
+
from .fields.uuids import sha2_uuid
|
|
14
13
|
|
|
15
14
|
|
|
16
15
|
def _get_database_path():
|
|
@@ -109,7 +108,7 @@ def _get_android_uuid():
|
|
|
109
108
|
def _do_salted_hash(value):
|
|
110
109
|
if not value:
|
|
111
110
|
return ""
|
|
112
|
-
if not isinstance(value,
|
|
111
|
+
if not isinstance(value, str):
|
|
113
112
|
value = str(value)
|
|
114
113
|
try:
|
|
115
114
|
value = value.encode()
|
|
@@ -185,7 +184,7 @@ def _get_mac_address_flags(mac):
|
|
|
185
184
|
"""
|
|
186
185
|
See: https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local
|
|
187
186
|
"""
|
|
188
|
-
if isinstance(mac,
|
|
187
|
+
if isinstance(mac, int):
|
|
189
188
|
mac = _mac_int_to_ether(mac)
|
|
190
189
|
|
|
191
190
|
first_octet = int(mac[:2], base=16)
|
morango/proquint.py
CHANGED
|
@@ -4,7 +4,6 @@ The simplest ways to use this module are the :func:`humanize` and :func:`uuid`
|
|
|
4
4
|
functions. For tighter control over the output, see :class:`HumanHasher`.
|
|
5
5
|
"""
|
|
6
6
|
import uuid
|
|
7
|
-
from django.utils import six
|
|
8
7
|
|
|
9
8
|
# Copyright (c) 2014 SUNET. All rights reserved.
|
|
10
9
|
#
|
|
@@ -59,7 +58,7 @@ def from_int(data):
|
|
|
59
58
|
:type data: int
|
|
60
59
|
:rtype: string
|
|
61
60
|
"""
|
|
62
|
-
if not isinstance(data,
|
|
61
|
+
if not isinstance(data, int):
|
|
63
62
|
raise TypeError("Input must be integer")
|
|
64
63
|
|
|
65
64
|
res = []
|
|
@@ -84,7 +83,7 @@ def to_int(data):
|
|
|
84
83
|
:type data: string
|
|
85
84
|
:rtype: int
|
|
86
85
|
"""
|
|
87
|
-
if not isinstance(data,
|
|
86
|
+
if not isinstance(data, str):
|
|
88
87
|
raise TypeError("Input must be string")
|
|
89
88
|
|
|
90
89
|
res = 0
|
morango/registry.py
CHANGED
|
@@ -7,7 +7,6 @@ import sys
|
|
|
7
7
|
from collections import OrderedDict
|
|
8
8
|
|
|
9
9
|
from django.db.models.fields.related import ForeignKey
|
|
10
|
-
from django.utils import six
|
|
11
10
|
|
|
12
11
|
from morango.constants import transfer_stages
|
|
13
12
|
from morango.errors import InvalidMorangoModelConfiguration
|
|
@@ -19,7 +18,7 @@ from morango.utils import SETTINGS
|
|
|
19
18
|
|
|
20
19
|
def _get_foreign_key_classes(m):
|
|
21
20
|
return set(
|
|
22
|
-
[field.
|
|
21
|
+
[field.related_model for field in m._meta.fields if isinstance(field, ForeignKey)]
|
|
23
22
|
)
|
|
24
23
|
|
|
25
24
|
|
|
@@ -36,6 +35,24 @@ def _multiple_self_ref_fk_check(class_model):
|
|
|
36
35
|
return False
|
|
37
36
|
|
|
38
37
|
|
|
38
|
+
def _check_manager(name, objects):
|
|
39
|
+
from morango.models.manager import SyncableModelManager
|
|
40
|
+
from morango.models.query import SyncableModelQuerySet
|
|
41
|
+
# syncable model checks
|
|
42
|
+
if not isinstance(objects, SyncableModelManager):
|
|
43
|
+
raise InvalidMorangoModelConfiguration(
|
|
44
|
+
"Manager for {} must inherit from SyncableModelManager.".format(
|
|
45
|
+
name
|
|
46
|
+
)
|
|
47
|
+
)
|
|
48
|
+
if not isinstance(objects.none(), SyncableModelQuerySet):
|
|
49
|
+
raise InvalidMorangoModelConfiguration(
|
|
50
|
+
"Queryset for {} model must inherit from SyncableModelQuerySet.".format(
|
|
51
|
+
name
|
|
52
|
+
)
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
39
56
|
class SyncableModelRegistry(object):
|
|
40
57
|
def __init__(self):
|
|
41
58
|
self.profile_models = {}
|
|
@@ -99,8 +116,6 @@ class SyncableModelRegistry(object):
|
|
|
99
116
|
|
|
100
117
|
import django.apps
|
|
101
118
|
from morango.models.core import SyncableModel
|
|
102
|
-
from morango.models.manager import SyncableModelManager
|
|
103
|
-
from morango.models.query import SyncableModelQuerySet
|
|
104
119
|
|
|
105
120
|
model_list = []
|
|
106
121
|
for model in django.apps.apps.get_models():
|
|
@@ -111,49 +126,10 @@ class SyncableModelRegistry(object):
|
|
|
111
126
|
raise InvalidMorangoModelConfiguration(
|
|
112
127
|
"Syncing models with more than 1 self referential ForeignKey is not supported."
|
|
113
128
|
)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
MorangoMPTTModel,
|
|
118
|
-
MorangoMPTTTreeManager,
|
|
119
|
-
MorangoTreeQuerySet,
|
|
120
|
-
)
|
|
129
|
+
# Check both the objects and the syncing_objects querysets.
|
|
130
|
+
_check_manager(name, model.objects)
|
|
131
|
+
_check_manager(name, model.syncing_objects)
|
|
121
132
|
|
|
122
|
-
# mptt syncable model checks
|
|
123
|
-
if issubclass(model, models.MPTTModel):
|
|
124
|
-
if not issubclass(model, MorangoMPTTModel):
|
|
125
|
-
raise InvalidMorangoModelConfiguration(
|
|
126
|
-
"{} that inherits from MPTTModel, should instead inherit from MorangoMPTTModel.".format(
|
|
127
|
-
name
|
|
128
|
-
)
|
|
129
|
-
)
|
|
130
|
-
if not isinstance(model.objects, MorangoMPTTTreeManager):
|
|
131
|
-
raise InvalidMorangoModelConfiguration(
|
|
132
|
-
"Manager for {} must inherit from MorangoMPTTTreeManager.".format(
|
|
133
|
-
name
|
|
134
|
-
)
|
|
135
|
-
)
|
|
136
|
-
if not isinstance(model.objects.none(), MorangoTreeQuerySet):
|
|
137
|
-
raise InvalidMorangoModelConfiguration(
|
|
138
|
-
"Queryset for {} model must inherit from MorangoTreeQuerySet.".format(
|
|
139
|
-
name
|
|
140
|
-
)
|
|
141
|
-
)
|
|
142
|
-
except ImportError:
|
|
143
|
-
pass
|
|
144
|
-
# syncable model checks
|
|
145
|
-
if not isinstance(model.objects, SyncableModelManager):
|
|
146
|
-
raise InvalidMorangoModelConfiguration(
|
|
147
|
-
"Manager for {} must inherit from SyncableModelManager.".format(
|
|
148
|
-
name
|
|
149
|
-
)
|
|
150
|
-
)
|
|
151
|
-
if not isinstance(model.objects.none(), SyncableModelQuerySet):
|
|
152
|
-
raise InvalidMorangoModelConfiguration(
|
|
153
|
-
"Queryset for {} model must inherit from SyncableModelQuerySet.".format(
|
|
154
|
-
name
|
|
155
|
-
)
|
|
156
|
-
)
|
|
157
133
|
if model._meta.many_to_many:
|
|
158
134
|
raise UnsupportedFieldType(
|
|
159
135
|
"{} model with a ManyToManyField is not supported in morango."
|
|
@@ -178,7 +154,7 @@ class SyncableModelRegistry(object):
|
|
|
178
154
|
self._insert_model_in_dependency_order(model, profile)
|
|
179
155
|
|
|
180
156
|
# for each profile, create a dict mapping from morango_model_name to model class
|
|
181
|
-
for profile, model_list in
|
|
157
|
+
for profile, model_list in self.profile_models.items():
|
|
182
158
|
mapping = OrderedDict()
|
|
183
159
|
for model in model_list:
|
|
184
160
|
mapping[model.morango_model_name] = model
|
morango/sync/backends/utils.py
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import sqlite3
|
|
2
|
+
from functools import lru_cache
|
|
2
3
|
from importlib import import_module
|
|
3
4
|
|
|
4
|
-
from django.utils.lru_cache import lru_cache
|
|
5
|
-
|
|
6
5
|
from morango.errors import MorangoError
|
|
7
6
|
|
|
8
7
|
|
|
@@ -110,15 +109,15 @@ class TemporaryTable(object):
|
|
|
110
109
|
"""
|
|
111
110
|
fields = []
|
|
112
111
|
params = []
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
112
|
+
schema_editor = self.connection.schema_editor()
|
|
113
|
+
for field in self.fields:
|
|
114
|
+
# generates the SQL expression for the table column
|
|
115
|
+
field_sql, field_params = schema_editor.column_sql(
|
|
116
|
+
self, field, include_default=True
|
|
117
|
+
)
|
|
118
|
+
field_sql_name = self.connection.ops.quote_name(field.column)
|
|
119
|
+
fields.append("{name} {sql}".format(name=field_sql_name, sql=field_sql))
|
|
120
|
+
params.extend(field_params)
|
|
122
121
|
with self.connection.cursor() as c:
|
|
123
122
|
self.backend._create_temporary_table(c, self.sql_name, fields, params)
|
|
124
123
|
|
morango/sync/context.py
CHANGED
|
@@ -63,6 +63,13 @@ class SessionContext(object):
|
|
|
63
63
|
"""
|
|
64
64
|
return self
|
|
65
65
|
|
|
66
|
+
def join(self, context):
|
|
67
|
+
"""
|
|
68
|
+
Perform any processing of session context after passing it to the middleware, which will
|
|
69
|
+
receive the result returned by `prepare` after that has happened
|
|
70
|
+
"""
|
|
71
|
+
pass
|
|
72
|
+
|
|
66
73
|
def update(
|
|
67
74
|
self,
|
|
68
75
|
transfer_session=None,
|
|
@@ -83,7 +90,7 @@ class SessionContext(object):
|
|
|
83
90
|
:type capabilities: str[]|None
|
|
84
91
|
:type error: BaseException|None
|
|
85
92
|
"""
|
|
86
|
-
if transfer_session and self.transfer_session:
|
|
93
|
+
if transfer_session and self.transfer_session and transfer_session.id != self.transfer_session.id:
|
|
87
94
|
raise MorangoContextUpdateError("Transfer session already exists")
|
|
88
95
|
elif (
|
|
89
96
|
transfer_session
|
|
@@ -92,14 +99,18 @@ class SessionContext(object):
|
|
|
92
99
|
):
|
|
93
100
|
raise MorangoContextUpdateError("Sync session mismatch")
|
|
94
101
|
|
|
95
|
-
if sync_filter and self.filter:
|
|
96
|
-
|
|
102
|
+
if sync_filter and self.filter and sync_filter != self.filter:
|
|
103
|
+
if not self.filter.is_subset_of(sync_filter):
|
|
104
|
+
raise MorangoContextUpdateError("The existing filter must be a subset of the new filter")
|
|
105
|
+
if transfer_stages.stage(self.stage) > transfer_stages.stage(transfer_stages.INITIALIZING):
|
|
106
|
+
raise MorangoContextUpdateError("Cannot update filter after initializing stage")
|
|
97
107
|
|
|
98
108
|
if is_push is not None and self.is_push is not None:
|
|
99
109
|
raise MorangoContextUpdateError("Push/pull method already exists")
|
|
100
110
|
|
|
101
111
|
self.transfer_session = transfer_session or self.transfer_session
|
|
102
|
-
|
|
112
|
+
if sync_filter or self.filter:
|
|
113
|
+
self.filter = Filter.add(self.filter, sync_filter)
|
|
103
114
|
self.is_push = is_push if is_push is not None else self.is_push
|
|
104
115
|
self.capabilities = set(capabilities or self.capabilities) & CAPABILITIES
|
|
105
116
|
self.update_state(stage=stage, stage_status=stage_status)
|
|
@@ -402,13 +413,40 @@ class CompositeSessionContext(SessionContext):
|
|
|
402
413
|
"""
|
|
403
414
|
return self.children[self._counter % len(self.children)]
|
|
404
415
|
|
|
416
|
+
def join(self, context):
|
|
417
|
+
"""
|
|
418
|
+
This updates some context attributes that really only make sense logically for what an
|
|
419
|
+
operation might change during a sync
|
|
420
|
+
|
|
421
|
+
:param context: The context that was returned previously from `prepare`
|
|
422
|
+
:type context: SessionContext
|
|
423
|
+
"""
|
|
424
|
+
updates = {}
|
|
425
|
+
if not self.transfer_session and context.transfer_session:
|
|
426
|
+
# if the transfer session is being resumed, we'd detect a different stage here,
|
|
427
|
+
# and thus we reset the counter, so we can be sure to start fresh at that stage
|
|
428
|
+
# on the next invocation of the middleware
|
|
429
|
+
if (
|
|
430
|
+
context.transfer_session.transfer_stage
|
|
431
|
+
and self._stage
|
|
432
|
+
and context.transfer_session.transfer_stage != self._stage
|
|
433
|
+
):
|
|
434
|
+
self._counter = 0
|
|
435
|
+
updates.update(
|
|
436
|
+
stage=context.transfer_session.transfer_stage,
|
|
437
|
+
stage_status=transfer_statuses.PENDING,
|
|
438
|
+
)
|
|
439
|
+
updates.update(transfer_session=context.transfer_session)
|
|
440
|
+
if self.filter != context.filter:
|
|
441
|
+
updates.update(sync_filter=context.filter)
|
|
442
|
+
if self.capabilities != context.capabilities:
|
|
443
|
+
updates.update(capabilities=context.capabilities)
|
|
444
|
+
if self.error != context.error:
|
|
445
|
+
updates.update(error=context.error)
|
|
446
|
+
if updates:
|
|
447
|
+
self.update(**updates)
|
|
448
|
+
|
|
405
449
|
def update(self, stage=None, stage_status=None, **kwargs):
|
|
406
|
-
"""
|
|
407
|
-
Updates the context object and its state
|
|
408
|
-
:param stage: The str transfer stage
|
|
409
|
-
:param stage_status: The str transfer stage status
|
|
410
|
-
:param kwargs: Other arguments to update the context with
|
|
411
|
-
"""
|
|
412
450
|
# update ourselves, but exclude stage and stage_status
|
|
413
451
|
super(CompositeSessionContext, self).update(**kwargs)
|
|
414
452
|
# update children contexts directly, but exclude stage and stage_status
|
|
@@ -416,36 +454,6 @@ class CompositeSessionContext(SessionContext):
|
|
|
416
454
|
# handle state changes after updating children
|
|
417
455
|
self.update_state(stage=stage, stage_status=stage_status)
|
|
418
456
|
|
|
419
|
-
# During the initializing stage, we want to make sure to synchronize the transfer session
|
|
420
|
-
# object between the composite and children contexts, using whatever child context's
|
|
421
|
-
# transfer session object that was updated on the context during initialization
|
|
422
|
-
current_stage = stage or self._stage
|
|
423
|
-
if not self.transfer_session and current_stage == transfer_stages.INITIALIZING:
|
|
424
|
-
try:
|
|
425
|
-
transfer_session = next(
|
|
426
|
-
c.transfer_session for c in self.children if c.transfer_session
|
|
427
|
-
)
|
|
428
|
-
# prepare an updates dictionary, so we can update everything at once
|
|
429
|
-
updates = dict(transfer_session=transfer_session)
|
|
430
|
-
|
|
431
|
-
# if the transfer session is being resumed, we'd detect a different stage here,
|
|
432
|
-
# and thus we reset the counter, so we can be sure to start fresh at that stage
|
|
433
|
-
# on the next invocation of the middleware
|
|
434
|
-
if (
|
|
435
|
-
transfer_session.transfer_stage
|
|
436
|
-
and transfer_session.transfer_stage != current_stage
|
|
437
|
-
):
|
|
438
|
-
self._counter = 0
|
|
439
|
-
updates.update(
|
|
440
|
-
stage=transfer_session.transfer_stage,
|
|
441
|
-
stage_status=transfer_statuses.PENDING,
|
|
442
|
-
)
|
|
443
|
-
|
|
444
|
-
# recurse into update with transfer session and possibly state updates too
|
|
445
|
-
self.update(**updates)
|
|
446
|
-
except StopIteration:
|
|
447
|
-
pass
|
|
448
|
-
|
|
449
457
|
def update_state(self, stage=None, stage_status=None):
|
|
450
458
|
"""
|
|
451
459
|
Updates the state of the transfer
|
morango/sync/controller.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
import math
|
|
2
3
|
from time import sleep
|
|
3
4
|
|
|
4
5
|
from morango.constants import transfer_stages
|
|
@@ -207,11 +208,16 @@ class SessionController(object):
|
|
|
207
208
|
tries = 0
|
|
208
209
|
context = context or self.context
|
|
209
210
|
max_interval = max_interval or context.max_backoff_interval
|
|
211
|
+
# solve for the number of tries at which our sleep time will always be max_interval
|
|
212
|
+
max_interval_tries = math.ceil(math.log(max_interval / 0.3 + 1) / math.log(2))
|
|
210
213
|
|
|
211
214
|
while result not in transfer_statuses.FINISHED_STATES:
|
|
212
215
|
if tries > 0:
|
|
213
216
|
# exponential backoff up to max_interval
|
|
214
|
-
|
|
217
|
+
if tries >= max_interval_tries:
|
|
218
|
+
sleep(max_interval)
|
|
219
|
+
else:
|
|
220
|
+
sleep(0.3 * (2 ** tries - 1))
|
|
215
221
|
result = self.proceed_to(target_stage, context=context)
|
|
216
222
|
tries += 1
|
|
217
223
|
if callable(callback):
|
|
@@ -247,6 +253,9 @@ class SessionController(object):
|
|
|
247
253
|
# invoke the middleware with the prepared context
|
|
248
254
|
result = middleware(prepared_context)
|
|
249
255
|
|
|
256
|
+
# return the prepared context back
|
|
257
|
+
context.join(prepared_context)
|
|
258
|
+
|
|
250
259
|
# don't update stage result if context's stage was updated during operation
|
|
251
260
|
if context.stage == stage:
|
|
252
261
|
context.update(stage_status=result)
|
morango/sync/operations.py
CHANGED
|
@@ -145,7 +145,7 @@ def _serialize_into_store(profile, filter=None):
|
|
|
145
145
|
for model in syncable_models.get_models(profile):
|
|
146
146
|
new_store_records = []
|
|
147
147
|
new_rmc_records = []
|
|
148
|
-
klass_queryset = model.
|
|
148
|
+
klass_queryset = model.syncing_objects.filter(_morango_dirty_bit=True)
|
|
149
149
|
if prefix_condition:
|
|
150
150
|
klass_queryset = klass_queryset.filter(prefix_condition)
|
|
151
151
|
store_records_dict = Store.objects.in_bulk(
|
morango/sync/session.py
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
|
|
3
3
|
from requests import exceptions
|
|
4
|
+
from morango import __version__
|
|
4
5
|
from requests.sessions import Session
|
|
5
6
|
from requests.utils import super_len
|
|
6
7
|
from requests.packages.urllib3.util.url import parse_url
|
|
7
8
|
|
|
8
9
|
from morango.utils import serialize_capabilities_to_client_request
|
|
10
|
+
from morango.utils import SETTINGS
|
|
9
11
|
|
|
10
12
|
|
|
11
13
|
logger = logging.getLogger(__name__)
|
|
@@ -35,6 +37,15 @@ class SessionWrapper(Session):
|
|
|
35
37
|
bytes_sent = 0
|
|
36
38
|
bytes_received = 0
|
|
37
39
|
|
|
40
|
+
def __init__(self):
|
|
41
|
+
super(SessionWrapper, self).__init__()
|
|
42
|
+
user_agent_header = "morango/{}".format(__version__)
|
|
43
|
+
if SETTINGS.CUSTOM_INSTANCE_INFO is not None:
|
|
44
|
+
instances = list(SETTINGS.CUSTOM_INSTANCE_INFO)
|
|
45
|
+
if instances:
|
|
46
|
+
user_agent_header += " " + "{}/{}".format(instances[0], SETTINGS.CUSTOM_INSTANCE_INFO.get(instances[0]))
|
|
47
|
+
self.headers["User-Agent"] = "{} {}".format(user_agent_header, self.headers["User-Agent"])
|
|
48
|
+
|
|
38
49
|
def request(self, method, url, **kwargs):
|
|
39
50
|
response = None
|
|
40
51
|
try:
|
morango/sync/syncsession.py
CHANGED
|
@@ -7,15 +7,14 @@ import os
|
|
|
7
7
|
import socket
|
|
8
8
|
import uuid
|
|
9
9
|
from io import BytesIO
|
|
10
|
+
from urllib.parse import urljoin
|
|
11
|
+
from urllib.parse import urlparse
|
|
10
12
|
|
|
11
13
|
from django.utils import timezone
|
|
12
|
-
from django.utils.six import iteritems
|
|
13
|
-
from django.utils.six import raise_from
|
|
14
|
-
from django.utils.six.moves.urllib.parse import urljoin
|
|
15
|
-
from django.utils.six.moves.urllib.parse import urlparse
|
|
16
14
|
from requests.adapters import HTTPAdapter
|
|
17
15
|
from requests.exceptions import HTTPError
|
|
18
16
|
from requests.packages.urllib3.util.retry import Retry
|
|
17
|
+
from django.db import transaction, connection
|
|
19
18
|
|
|
20
19
|
from .session import SessionWrapper
|
|
21
20
|
from morango.api.serializers import CertificateSerializer
|
|
@@ -30,9 +29,11 @@ from morango.errors import MorangoError
|
|
|
30
29
|
from morango.errors import MorangoResumeSyncError
|
|
31
30
|
from morango.errors import MorangoServerDoesNotAllowNewCertPush
|
|
32
31
|
from morango.models.certificates import Certificate
|
|
32
|
+
from morango.models.certificates import Filter
|
|
33
33
|
from morango.models.certificates import Key
|
|
34
34
|
from morango.models.core import InstanceIDModel
|
|
35
35
|
from morango.models.core import SyncSession
|
|
36
|
+
from morango.sync.backends.utils import load_backend
|
|
36
37
|
from morango.sync.context import CompositeSessionContext
|
|
37
38
|
from morango.sync.context import LocalSessionContext
|
|
38
39
|
from morango.sync.context import NetworkSessionContext
|
|
@@ -41,6 +42,7 @@ from morango.sync.utils import SyncSignal
|
|
|
41
42
|
from morango.sync.utils import SyncSignalGroup
|
|
42
43
|
from morango.utils import CAPABILITIES
|
|
43
44
|
from morango.utils import pid_exists
|
|
45
|
+
from morango.sync.utils import lock_partitions
|
|
44
46
|
|
|
45
47
|
if GZIP_BUFFER_POST in CAPABILITIES:
|
|
46
48
|
from gzip import GzipFile
|
|
@@ -48,6 +50,7 @@ if GZIP_BUFFER_POST in CAPABILITIES:
|
|
|
48
50
|
|
|
49
51
|
logger = logging.getLogger(__name__)
|
|
50
52
|
|
|
53
|
+
DBBackend = load_backend(connection)
|
|
51
54
|
|
|
52
55
|
def _join_with_logical_operator(lst, operator):
|
|
53
56
|
op = ") {operator} (".format(operator=operator)
|
|
@@ -225,6 +228,14 @@ class NetworkSyncConnection(Connection):
|
|
|
225
228
|
if not server_cert.verify(message, session_resp.json().get("signature")):
|
|
226
229
|
raise CertificateSignatureInvalid()
|
|
227
230
|
|
|
231
|
+
client_instance = InstanceIDModel.get_or_create_current_instance()[0]
|
|
232
|
+
|
|
233
|
+
server_instance_json = session_resp.json().get("server_instance") or "{}"
|
|
234
|
+
server_instance_id = None
|
|
235
|
+
if server_instance_json:
|
|
236
|
+
server_instance = json.loads(server_instance_json)
|
|
237
|
+
server_instance_id = server_instance.get("id")
|
|
238
|
+
|
|
228
239
|
# build the data to be used for creating our own syncsession
|
|
229
240
|
data = {
|
|
230
241
|
"id": data["id"],
|
|
@@ -239,23 +250,29 @@ class NetworkSyncConnection(Connection):
|
|
|
239
250
|
"connection_path": self.base_url,
|
|
240
251
|
"client_ip": data["client_ip"],
|
|
241
252
|
"server_ip": data["server_ip"],
|
|
242
|
-
"
|
|
253
|
+
"client_instance_id": client_instance.id,
|
|
254
|
+
"client_instance_json": json.dumps(
|
|
243
255
|
InstanceIDSerializer(
|
|
244
|
-
|
|
256
|
+
client_instance
|
|
245
257
|
).data
|
|
246
258
|
),
|
|
247
|
-
"
|
|
259
|
+
"server_instance_id": server_instance_id,
|
|
260
|
+
"server_instance_json": session_resp.json().get("server_instance") or "{}",
|
|
248
261
|
"process_id": os.getpid(),
|
|
249
262
|
}
|
|
250
263
|
sync_session = SyncSession.objects.create(**data)
|
|
251
264
|
return SyncSessionClient(self, sync_session)
|
|
252
265
|
|
|
253
|
-
def resume_sync_session(self, sync_session_id, chunk_size=None):
|
|
266
|
+
def resume_sync_session(self, sync_session_id, chunk_size=None, ignore_existing_process=False):
|
|
254
267
|
"""
|
|
255
268
|
Resumes an existing sync session given an ID
|
|
256
269
|
|
|
257
270
|
:param sync_session_id: The UUID of the `SyncSession` to resume
|
|
258
271
|
:param chunk_size: An optional parameter specifying the size for each transferred chunk
|
|
272
|
+
:type chunk_size: int
|
|
273
|
+
:param ignore_existing_process:An optional parameter specifying whether to ignore an
|
|
274
|
+
existing active process ID
|
|
275
|
+
:type ignore_existing_process: bool
|
|
259
276
|
:return: A SyncSessionClient instance
|
|
260
277
|
:rtype: SyncSessionClient
|
|
261
278
|
"""
|
|
@@ -271,7 +288,8 @@ class NetworkSyncConnection(Connection):
|
|
|
271
288
|
|
|
272
289
|
# check that process of existing session isn't still running
|
|
273
290
|
if (
|
|
274
|
-
|
|
291
|
+
not ignore_existing_process
|
|
292
|
+
and sync_session.process_id
|
|
275
293
|
and sync_session.process_id != os.getpid()
|
|
276
294
|
and pid_exists(sync_session.process_id)
|
|
277
295
|
):
|
|
@@ -284,7 +302,7 @@ class NetworkSyncConnection(Connection):
|
|
|
284
302
|
try:
|
|
285
303
|
self._get_sync_session(sync_session)
|
|
286
304
|
except HTTPError as e:
|
|
287
|
-
|
|
305
|
+
raise MorangoResumeSyncError("Failure resuming sync session") from e
|
|
288
306
|
|
|
289
307
|
# update process id
|
|
290
308
|
sync_session.process_id = os.getpid()
|
|
@@ -338,11 +356,15 @@ class NetworkSyncConnection(Connection):
|
|
|
338
356
|
cert_chain_response = self._get_certificate_chain(
|
|
339
357
|
params={"ancestors_of": parent_cert.id}
|
|
340
358
|
)
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
359
|
+
cert_chain = cert_chain_response.json()
|
|
360
|
+
with transaction.atomic():
|
|
361
|
+
lock_partitions(DBBackend, sync_filter=Filter(cert_chain[0]["id"]))
|
|
362
|
+
# check again, now that we have a lock
|
|
363
|
+
if not Certificate.objects.filter(id=parent_cert.id).exists():
|
|
364
|
+
# upon receiving cert chain from server, we attempt to save the chain into our records
|
|
365
|
+
Certificate.save_certificate_chain(
|
|
366
|
+
cert_chain, expected_last_id=parent_cert.id
|
|
367
|
+
)
|
|
346
368
|
|
|
347
369
|
csr_key = Key()
|
|
348
370
|
# build up data for csr
|
|
@@ -425,7 +447,7 @@ class NetworkSyncConnection(Connection):
|
|
|
425
447
|
# convert user arguments into query str for passing to auth layer
|
|
426
448
|
if isinstance(userargs, dict):
|
|
427
449
|
userargs = "&".join(
|
|
428
|
-
["{}={}".format(key, val) for (key, val) in
|
|
450
|
+
["{}={}".format(key, val) for (key, val) in userargs.items()]
|
|
429
451
|
)
|
|
430
452
|
return self.session.post(
|
|
431
453
|
self.urlresolve(api_urls.CERTIFICATE), json=data, auth=(userargs, password)
|
|
@@ -612,12 +634,9 @@ class TransferClient(object):
|
|
|
612
634
|
"""
|
|
613
635
|
result = self.controller.proceed_to_and_wait_for(stage, callback=callback)
|
|
614
636
|
if result == transfer_statuses.ERRORED:
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
),
|
|
619
|
-
self.context.error,
|
|
620
|
-
)
|
|
637
|
+
raise MorangoError(
|
|
638
|
+
error_msg or "Stage `{}` failed".format(self.context.stage)
|
|
639
|
+
) from self.context.error
|
|
621
640
|
|
|
622
641
|
def initialize(self, sync_filter):
|
|
623
642
|
"""
|
morango/urls.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from django.
|
|
2
|
-
from django.
|
|
1
|
+
from django.urls import include
|
|
2
|
+
from django.urls import path
|
|
3
3
|
|
|
4
|
-
urlpatterns = [
|
|
4
|
+
urlpatterns = [path("api/morango/v1/", include("morango.api.urls"))]
|
morango/utils.py
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import os
|
|
2
|
-
import six
|
|
3
2
|
from importlib import import_module
|
|
4
3
|
|
|
5
4
|
from django.conf import settings
|
|
6
5
|
|
|
7
6
|
from morango.constants import settings as default_settings
|
|
8
7
|
from morango.constants.capabilities import ALLOW_CERTIFICATE_PUSHING
|
|
9
|
-
from morango.constants.capabilities import GZIP_BUFFER_POST
|
|
10
8
|
from morango.constants.capabilities import ASYNC_OPERATIONS
|
|
11
9
|
from morango.constants.capabilities import FSIC_V2_FORMAT
|
|
10
|
+
from morango.constants.capabilities import GZIP_BUFFER_POST
|
|
12
11
|
|
|
13
12
|
|
|
14
13
|
def do_import(import_string):
|
|
@@ -32,7 +31,7 @@ class Settings(object):
|
|
|
32
31
|
def __getattribute__(self, key):
|
|
33
32
|
"""Coalesces settings with the defaults"""
|
|
34
33
|
value = getattr(settings, key, getattr(default_settings, key, None))
|
|
35
|
-
if key == "MORANGO_INSTANCE_INFO" and isinstance(value,
|
|
34
|
+
if key == "MORANGO_INSTANCE_INFO" and isinstance(value, str):
|
|
36
35
|
value = dict(do_import(value))
|
|
37
36
|
return value
|
|
38
37
|
|