diffsync 1.10.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {diffsync-1.10.0 → diffsync-2.0.0}/CHANGELOG.md +27 -17
- {diffsync-1.10.0 → diffsync-2.0.0}/PKG-INFO +3 -3
- {diffsync-1.10.0 → diffsync-2.0.0}/README.md +1 -1
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/__init__.py +93 -48
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/helpers.py +6 -6
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/store/__init__.py +7 -7
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/store/local.py +2 -2
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/store/redis.py +3 -3
- {diffsync-1.10.0 → diffsync-2.0.0}/pyproject.toml +2 -2
- {diffsync-1.10.0 → diffsync-2.0.0}/LICENSE +0 -0
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/diff.py +0 -0
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/enum.py +0 -0
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/exceptions.py +0 -0
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/logging.py +0 -0
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/py.typed +0 -0
- {diffsync-1.10.0 → diffsync-2.0.0}/diffsync/utils.py +0 -0
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
|
6
|
+
|
|
7
|
+
## [2.0.0]
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
|
|
11
|
+
- **BREAKING CHANGE** #236/240 - Upgrade to Pydantic v2.
|
|
12
|
+
|
|
13
|
+
## [1.10.0] - 2023-11-16
|
|
4
14
|
|
|
5
15
|
### Fixed
|
|
6
16
|
|
|
@@ -11,7 +21,7 @@
|
|
|
11
21
|
|
|
12
22
|
- #247 - Deprecates Python 3.7
|
|
13
23
|
|
|
14
|
-
##
|
|
24
|
+
## [1.9.0] - 2023-10-16
|
|
15
25
|
|
|
16
26
|
### Added
|
|
17
27
|
|
|
@@ -21,7 +31,7 @@
|
|
|
21
31
|
|
|
22
32
|
- #219 - Type hinting overhaul
|
|
23
33
|
|
|
24
|
-
##
|
|
34
|
+
## [1.8.0] - 2023-04-18
|
|
25
35
|
|
|
26
36
|
### Added
|
|
27
37
|
|
|
@@ -34,7 +44,7 @@
|
|
|
34
44
|
- #77/#188 - `sync_from()` and `sync_to()` now return the `Diff` that was applied.
|
|
35
45
|
- #211 - Loosened `packaging` and `structlog` library dependency constraints for broader compatibility.
|
|
36
46
|
|
|
37
|
-
##
|
|
47
|
+
## [1.7.0] - 2022-11-03
|
|
38
48
|
|
|
39
49
|
### Changed
|
|
40
50
|
|
|
@@ -52,15 +62,15 @@
|
|
|
52
62
|
|
|
53
63
|
### Fixed
|
|
54
64
|
|
|
55
|
-
- #149 Limit redundant CI concurrency
|
|
65
|
+
- #149 - Limit redundant CI concurrency
|
|
56
66
|
|
|
57
|
-
##
|
|
67
|
+
## [1.6.0] - 2022-07-09
|
|
58
68
|
|
|
59
69
|
### Changed
|
|
60
70
|
|
|
61
71
|
- #120 - Dropped support for Python 3.6, new minimum is Python 3.7
|
|
62
72
|
|
|
63
|
-
##
|
|
73
|
+
## [1.5.1] - 2022-06-30
|
|
64
74
|
|
|
65
75
|
### Added
|
|
66
76
|
|
|
@@ -75,13 +85,13 @@
|
|
|
75
85
|
- #115 - Fixed ReadTheDocs rendering pipeline
|
|
76
86
|
- #118 - Fixed a regression in `DiffSync.get(modelname, identifiers)` introduced in 1.5.0
|
|
77
87
|
|
|
78
|
-
##
|
|
88
|
+
## [1.5.0] - 2022-06-07
|
|
79
89
|
|
|
80
90
|
### Added
|
|
81
91
|
|
|
82
92
|
- #106 - Add a new, optional, backend store based in Redis
|
|
83
93
|
|
|
84
|
-
##
|
|
94
|
+
## [1.4.3] - 2022-03-03
|
|
85
95
|
|
|
86
96
|
### Fixed
|
|
87
97
|
|
|
@@ -91,9 +101,9 @@
|
|
|
91
101
|
|
|
92
102
|
### Changed
|
|
93
103
|
|
|
94
|
-
- #103 Update development dependencies
|
|
104
|
+
- #103 - Update development dependencies
|
|
95
105
|
|
|
96
|
-
##
|
|
106
|
+
## [1.4.2] - 2022-02-28
|
|
97
107
|
|
|
98
108
|
**WARNING** - #90 inadvertently introduced a breaking API change in DiffSync 1.4.0 through 1.4.2 (#101); this change was reverted in #102 for DiffSync 1.4.3 and later. We recommend not using this release, and moving to 1.4.3 instead.
|
|
99
109
|
|
|
@@ -101,7 +111,7 @@
|
|
|
101
111
|
|
|
102
112
|
- #100 - Added explicit dependency on `packaging`.
|
|
103
113
|
|
|
104
|
-
##
|
|
114
|
+
## [1.4.1] - 2022-01-26
|
|
105
115
|
|
|
106
116
|
**WARNING** - #90 inadvertently introduced a breaking API change in DiffSync 1.4.0 through 1.4.2 (#101); this change was reverted in #102 for DiffSync 1.4.3 and later. We recommend not using this release, and moving to 1.4.3 instead.
|
|
107
117
|
|
|
@@ -109,7 +119,7 @@
|
|
|
109
119
|
|
|
110
120
|
- #95 - Removed optional dependencies on `sphinx`, `m2r2`, `sphinx-rtd-theme`, `toml`.
|
|
111
121
|
|
|
112
|
-
##
|
|
122
|
+
## [1.4.0] - 2022-01-24
|
|
113
123
|
|
|
114
124
|
**WARNING** - #90 inadvertently introduced a breaking API change in DiffSync 1.4.0 through 1.4.2 (#101); this change was reverted in #102 for DiffSync 1.4.3 and later. We recommend not using this release, and moving to 1.4.3 instead.
|
|
115
125
|
|
|
@@ -138,19 +148,19 @@
|
|
|
138
148
|
- #51 - Update minimum Pydantic version due to security advisory GHSA-5jqp-qgf6-3pvh
|
|
139
149
|
- #63 - Fix type in Readme
|
|
140
150
|
|
|
141
|
-
##
|
|
151
|
+
## [1.3.0] - 2021-04-07
|
|
142
152
|
|
|
143
153
|
### Added
|
|
144
154
|
|
|
145
155
|
- #48 - added optional `callback` argument to `diff_from`/`diff_to`/`sync_from`/`sync_to` for use with progress reporting.
|
|
146
156
|
|
|
147
|
-
##
|
|
157
|
+
## [1.2.0] - 2020-12-08
|
|
148
158
|
|
|
149
159
|
### Added
|
|
150
160
|
|
|
151
161
|
- #45 - minimum Python version lowered from 3.7 to 3.6, also now tested against Python 3.9.
|
|
152
162
|
|
|
153
|
-
##
|
|
163
|
+
## [1.1.0] - 2020-12-01
|
|
154
164
|
|
|
155
165
|
### Added
|
|
156
166
|
|
|
@@ -168,6 +178,6 @@
|
|
|
168
178
|
|
|
169
179
|
- #44 - On CRUD failure, do not generate an extraneous "success" log message in addition to the "failed" message
|
|
170
180
|
|
|
171
|
-
##
|
|
181
|
+
## [1.0.0] - 2020-10-23
|
|
172
182
|
|
|
173
183
|
Initial release
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: diffsync
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: Library to easily sync/diff/update 2 different data sources
|
|
5
5
|
Home-page: https://diffsync.readthedocs.io
|
|
6
6
|
License: Apache-2.0
|
|
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
18
18
|
Provides-Extra: redis
|
|
19
19
|
Requires-Dist: colorama (>=0.4.3,<0.5.0)
|
|
20
20
|
Requires-Dist: packaging (>=21.3,<24.0)
|
|
21
|
-
Requires-Dist: pydantic (>=
|
|
21
|
+
Requires-Dist: pydantic (>=2.0.0,<3.0.0)
|
|
22
22
|
Requires-Dist: redis (>=4.3,<5.0) ; extra == "redis"
|
|
23
23
|
Requires-Dist: structlog (>=20.1.0,<23.0.0)
|
|
24
24
|
Requires-Dist: typing-extensions (>=4.0.1) ; python_version < "3.11"
|
|
@@ -43,7 +43,7 @@ DiffSync is at its most useful when you have multiple sources or sets of data to
|
|
|
43
43
|
|
|
44
44
|
# Overview of DiffSync
|
|
45
45
|
|
|
46
|
-
DiffSync acts as an intermediate translation layer between all of the data sets you are diffing and/or syncing. In practical terms, this means that to use DiffSync, you will define a set of data models as well as the “adapters” needed to translate between each base data source and the data model. In Python terms, the adapters will be subclasses of the `
|
|
46
|
+
DiffSync acts as an intermediate translation layer between all of the data sets you are diffing and/or syncing. In practical terms, this means that to use DiffSync, you will define a set of data models as well as the “adapters” needed to translate between each base data source and the data model. In Python terms, the adapters will be subclasses of the `Adapter` class, and each data model class will be a subclass of the `DiffSyncModel` class.
|
|
47
47
|
|
|
48
48
|

|
|
49
49
|
|
|
@@ -15,7 +15,7 @@ DiffSync is at its most useful when you have multiple sources or sets of data to
|
|
|
15
15
|
|
|
16
16
|
# Overview of DiffSync
|
|
17
17
|
|
|
18
|
-
DiffSync acts as an intermediate translation layer between all of the data sets you are diffing and/or syncing. In practical terms, this means that to use DiffSync, you will define a set of data models as well as the “adapters” needed to translate between each base data source and the data model. In Python terms, the adapters will be subclasses of the `
|
|
18
|
+
DiffSync acts as an intermediate translation layer between all of the data sets you are diffing and/or syncing. In practical terms, this means that to use DiffSync, you will define a set of data models as well as the “adapters” needed to translate between each base data source and the data model. In Python terms, the adapters will be subclasses of the `Adapter` class, and each data model class will be a subclass of the `DiffSyncModel` class.
|
|
19
19
|
|
|
20
20
|

|
|
21
21
|
|
|
@@ -16,14 +16,31 @@ limitations under the License.
|
|
|
16
16
|
"""
|
|
17
17
|
import sys
|
|
18
18
|
from inspect import isclass
|
|
19
|
-
from typing import
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
from typing import (
|
|
20
|
+
Callable,
|
|
21
|
+
ClassVar,
|
|
22
|
+
Dict,
|
|
23
|
+
List,
|
|
24
|
+
Optional,
|
|
25
|
+
Tuple,
|
|
26
|
+
Type,
|
|
27
|
+
Union,
|
|
28
|
+
Any,
|
|
29
|
+
Set,
|
|
30
|
+
)
|
|
31
|
+
import warnings
|
|
32
|
+
|
|
33
|
+
from pydantic import ConfigDict, BaseModel, PrivateAttr
|
|
22
34
|
import structlog # type: ignore
|
|
23
35
|
|
|
24
36
|
from diffsync.diff import Diff
|
|
25
37
|
from diffsync.enum import DiffSyncModelFlags, DiffSyncFlags, DiffSyncStatus
|
|
26
|
-
from diffsync.exceptions import
|
|
38
|
+
from diffsync.exceptions import (
|
|
39
|
+
DiffClassMismatch,
|
|
40
|
+
ObjectAlreadyExists,
|
|
41
|
+
ObjectStoreWrongType,
|
|
42
|
+
ObjectNotFound,
|
|
43
|
+
)
|
|
27
44
|
from diffsync.helpers import DiffSyncDiffer, DiffSyncSyncer
|
|
28
45
|
from diffsync.store import BaseStore
|
|
29
46
|
from diffsync.store.local import LocalStore
|
|
@@ -96,8 +113,8 @@ class DiffSyncModel(BaseModel):
|
|
|
96
113
|
Can be set as a class attribute or an instance attribute as needed.
|
|
97
114
|
"""
|
|
98
115
|
|
|
99
|
-
|
|
100
|
-
"""Optional: the
|
|
116
|
+
adapter: Optional["Adapter"] = None
|
|
117
|
+
"""Optional: the Adapter instance that owns this model instance."""
|
|
101
118
|
|
|
102
119
|
_status: DiffSyncStatus = PrivateAttr(DiffSyncStatus.SUCCESS)
|
|
103
120
|
"""Status of the last attempt at creating/updating/deleting this model."""
|
|
@@ -105,30 +122,27 @@ class DiffSyncModel(BaseModel):
|
|
|
105
122
|
_status_message: str = PrivateAttr("")
|
|
106
123
|
"""Message, if any, associated with the create/update/delete status value."""
|
|
107
124
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
# Let us have a DiffSync as an instance variable even though DiffSync is not a Pydantic model itself.
|
|
112
|
-
arbitrary_types_allowed = True
|
|
125
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
126
|
+
"""Pydantic-specific configuration to allow arbitrary types on this class."""
|
|
113
127
|
|
|
114
|
-
|
|
128
|
+
@classmethod
|
|
129
|
+
def __pydantic_init_subclass__(cls, **kwargs: Any) -> None:
|
|
115
130
|
"""Validate that the various class attribute declarations correspond to actual instance fields.
|
|
116
131
|
|
|
117
132
|
Called automatically on subclass declaration.
|
|
118
133
|
"""
|
|
119
|
-
variables = cls.__fields__.keys()
|
|
120
134
|
# Make sure that any field referenced by name actually exists on the model
|
|
121
135
|
for attr in cls._identifiers:
|
|
122
|
-
if attr not in
|
|
136
|
+
if attr not in cls.model_fields and not hasattr(cls, attr):
|
|
123
137
|
raise AttributeError(f"_identifiers {cls._identifiers} references missing or un-annotated attr {attr}")
|
|
124
138
|
for attr in cls._shortname:
|
|
125
|
-
if attr not in
|
|
139
|
+
if attr not in cls.model_fields:
|
|
126
140
|
raise AttributeError(f"_shortname {cls._shortname} references missing or un-annotated attr {attr}")
|
|
127
141
|
for attr in cls._attributes:
|
|
128
|
-
if attr not in
|
|
142
|
+
if attr not in cls.model_fields:
|
|
129
143
|
raise AttributeError(f"_attributes {cls._attributes} references missing or un-annotated attr {attr}")
|
|
130
144
|
for attr in cls._children.values():
|
|
131
|
-
if attr not in
|
|
145
|
+
if attr not in cls.model_fields:
|
|
132
146
|
raise AttributeError(f"_children {cls._children} references missing or un-annotated attr {attr}")
|
|
133
147
|
|
|
134
148
|
# Any given field can only be in one of (_identifiers, _attributes, _children)
|
|
@@ -149,18 +163,18 @@ class DiffSyncModel(BaseModel):
|
|
|
149
163
|
return self.get_unique_id()
|
|
150
164
|
|
|
151
165
|
def dict(self, **kwargs: Any) -> Dict:
|
|
152
|
-
"""Convert this DiffSyncModel to a dict, excluding the
|
|
166
|
+
"""Convert this DiffSyncModel to a dict, excluding the adapter field by default as it is not serializable."""
|
|
153
167
|
if "exclude" not in kwargs:
|
|
154
|
-
kwargs["exclude"] = {"
|
|
155
|
-
return super().
|
|
168
|
+
kwargs["exclude"] = {"adapter"}
|
|
169
|
+
return super().model_dump(**kwargs)
|
|
156
170
|
|
|
157
171
|
def json(self, **kwargs: Any) -> StrType:
|
|
158
|
-
"""Convert this DiffSyncModel to a JSON string, excluding the
|
|
172
|
+
"""Convert this DiffSyncModel to a JSON string, excluding the adapter field by default as it is not serializable."""
|
|
159
173
|
if "exclude" not in kwargs:
|
|
160
|
-
kwargs["exclude"] = {"
|
|
174
|
+
kwargs["exclude"] = {"adapter"}
|
|
161
175
|
if "exclude_defaults" not in kwargs:
|
|
162
176
|
kwargs["exclude_defaults"] = True
|
|
163
|
-
return super().
|
|
177
|
+
return super().model_dump_json(**kwargs)
|
|
164
178
|
|
|
165
179
|
def str(self, include_children: bool = True, indent: int = 0) -> StrType:
|
|
166
180
|
"""Build a detailed string representation of this DiffSyncModel and optionally its children."""
|
|
@@ -171,12 +185,12 @@ class DiffSyncModel(BaseModel):
|
|
|
171
185
|
child_ids = getattr(self, fieldname)
|
|
172
186
|
if not child_ids:
|
|
173
187
|
output += ": []"
|
|
174
|
-
elif not self.
|
|
188
|
+
elif not self.adapter or not include_children:
|
|
175
189
|
output += f": {child_ids}"
|
|
176
190
|
else:
|
|
177
191
|
for child_id in child_ids:
|
|
178
192
|
try:
|
|
179
|
-
child = self.
|
|
193
|
+
child = self.adapter.get(modelname, child_id)
|
|
180
194
|
output += "\n" + child.str(include_children=include_children, indent=indent + 4)
|
|
181
195
|
except ObjectNotFound:
|
|
182
196
|
output += f"\n{margin} {child_id} (ERROR: details unavailable)"
|
|
@@ -188,32 +202,32 @@ class DiffSyncModel(BaseModel):
|
|
|
188
202
|
self._status_message = message
|
|
189
203
|
|
|
190
204
|
@classmethod
|
|
191
|
-
def create_base(cls,
|
|
205
|
+
def create_base(cls, adapter: "Adapter", ids: Dict, attrs: Dict) -> Optional[Self]:
|
|
192
206
|
"""Instantiate this class, along with any platform-specific data creation.
|
|
193
207
|
|
|
194
208
|
This method is not meant to be subclassed, users should redefine create() instead.
|
|
195
209
|
|
|
196
210
|
Args:
|
|
197
|
-
|
|
211
|
+
adapter: The master data store for other DiffSyncModel instances that we might need to reference
|
|
198
212
|
ids: Dictionary of unique-identifiers needed to create the new object
|
|
199
213
|
attrs: Dictionary of additional attributes to set on the new object
|
|
200
214
|
|
|
201
215
|
Returns:
|
|
202
216
|
DiffSyncModel: instance of this class.
|
|
203
217
|
"""
|
|
204
|
-
model = cls(**ids,
|
|
218
|
+
model = cls(**ids, adapter=adapter, **attrs)
|
|
205
219
|
model.set_status(DiffSyncStatus.SUCCESS, "Created successfully")
|
|
206
220
|
return model
|
|
207
221
|
|
|
208
222
|
@classmethod
|
|
209
|
-
def create(cls,
|
|
223
|
+
def create(cls, adapter: "Adapter", ids: Dict, attrs: Dict) -> Optional[Self]:
|
|
210
224
|
"""Instantiate this class, along with any platform-specific data creation.
|
|
211
225
|
|
|
212
226
|
Subclasses must call `super().create()` or `self.create_base()`; they may wish to then override the default status information
|
|
213
227
|
by calling `set_status()` to provide more context (such as details of any interactions with underlying systems).
|
|
214
228
|
|
|
215
229
|
Args:
|
|
216
|
-
|
|
230
|
+
adapter: The master data store for other DiffSyncModel instances that we might need to reference
|
|
217
231
|
ids: Dictionary of unique-identifiers needed to create the new object
|
|
218
232
|
attrs: Dictionary of additional attributes to set on the new object
|
|
219
233
|
|
|
@@ -224,7 +238,7 @@ class DiffSyncModel(BaseModel):
|
|
|
224
238
|
Raises:
|
|
225
239
|
ObjectNotCreated: if an error occurred.
|
|
226
240
|
"""
|
|
227
|
-
return cls.create_base(
|
|
241
|
+
return cls.create_base(adapter=adapter, ids=ids, attrs=attrs)
|
|
228
242
|
|
|
229
243
|
def update_base(self, attrs: Dict) -> Optional[Self]:
|
|
230
244
|
"""Base Update method to update the attributes of this instance, along with any platform-specific data updates.
|
|
@@ -380,7 +394,10 @@ class DiffSyncModel(BaseModel):
|
|
|
380
394
|
attr_name = self._children[child_type]
|
|
381
395
|
childs = getattr(self, attr_name)
|
|
382
396
|
if child.get_unique_id() in childs:
|
|
383
|
-
raise ObjectAlreadyExists(
|
|
397
|
+
raise ObjectAlreadyExists(
|
|
398
|
+
f"Already storing a {child_type} with unique_id {child.get_unique_id()}",
|
|
399
|
+
child,
|
|
400
|
+
)
|
|
384
401
|
childs.append(child.get_unique_id())
|
|
385
402
|
|
|
386
403
|
def remove_child(self, child: "DiffSyncModel") -> None:
|
|
@@ -407,7 +424,7 @@ class DiffSyncModel(BaseModel):
|
|
|
407
424
|
childs.remove(child.get_unique_id())
|
|
408
425
|
|
|
409
426
|
|
|
410
|
-
class
|
|
427
|
+
class Adapter: # pylint: disable=too-many-public-methods
|
|
411
428
|
"""Class for storing a group of DiffSyncModel instances and diffing/synchronizing to another DiffSync instance."""
|
|
412
429
|
|
|
413
430
|
# In any subclass, you would add mapping of names to specific model classes here:
|
|
@@ -421,7 +438,9 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
421
438
|
"""List of top-level modelnames to begin from when diffing or synchronizing."""
|
|
422
439
|
|
|
423
440
|
def __init__(
|
|
424
|
-
self,
|
|
441
|
+
self,
|
|
442
|
+
name: Optional[str] = None,
|
|
443
|
+
internal_storage_engine: Union[Type[BaseStore], BaseStore] = LocalStore,
|
|
425
444
|
) -> None:
|
|
426
445
|
"""Generic initialization function.
|
|
427
446
|
|
|
@@ -430,9 +449,9 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
430
449
|
|
|
431
450
|
if isinstance(internal_storage_engine, BaseStore):
|
|
432
451
|
self.store = internal_storage_engine
|
|
433
|
-
self.store.
|
|
452
|
+
self.store.adapter = self
|
|
434
453
|
else:
|
|
435
|
-
self.store = internal_storage_engine(
|
|
454
|
+
self.store = internal_storage_engine(adapter=self)
|
|
436
455
|
|
|
437
456
|
# If the type is not defined, use the name of the class as the default value
|
|
438
457
|
if self.type is None:
|
|
@@ -540,7 +559,7 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
540
559
|
|
|
541
560
|
def sync_from( # pylint: disable=too-many-arguments
|
|
542
561
|
self,
|
|
543
|
-
source: "
|
|
562
|
+
source: "Adapter",
|
|
544
563
|
diff_class: Type[Diff] = Diff,
|
|
545
564
|
flags: DiffSyncFlags = DiffSyncFlags.NONE,
|
|
546
565
|
callback: Optional[Callable[[StrType, int, int], None]] = None,
|
|
@@ -569,7 +588,13 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
569
588
|
# Generate the diff if an existing diff was not provided
|
|
570
589
|
if not diff:
|
|
571
590
|
diff = self.diff_from(source, diff_class=diff_class, flags=flags, callback=callback)
|
|
572
|
-
syncer = DiffSyncSyncer(
|
|
591
|
+
syncer = DiffSyncSyncer(
|
|
592
|
+
diff=diff,
|
|
593
|
+
src_diffsync=source,
|
|
594
|
+
dst_diffsync=self,
|
|
595
|
+
flags=flags,
|
|
596
|
+
callback=callback,
|
|
597
|
+
)
|
|
573
598
|
result = syncer.perform_sync()
|
|
574
599
|
if result:
|
|
575
600
|
self.sync_complete(source, diff, flags, syncer.base_logger)
|
|
@@ -578,7 +603,7 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
578
603
|
|
|
579
604
|
def sync_to( # pylint: disable=too-many-arguments
|
|
580
605
|
self,
|
|
581
|
-
target: "
|
|
606
|
+
target: "Adapter",
|
|
582
607
|
diff_class: Type[Diff] = Diff,
|
|
583
608
|
flags: DiffSyncFlags = DiffSyncFlags.NONE,
|
|
584
609
|
callback: Optional[Callable[[StrType, int, int], None]] = None,
|
|
@@ -602,7 +627,7 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
602
627
|
|
|
603
628
|
def sync_complete(
|
|
604
629
|
self,
|
|
605
|
-
source: "
|
|
630
|
+
source: "Adapter",
|
|
606
631
|
diff: Diff,
|
|
607
632
|
flags: DiffSyncFlags = DiffSyncFlags.NONE,
|
|
608
633
|
logger: Optional[structlog.BoundLogger] = None,
|
|
@@ -628,7 +653,7 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
628
653
|
|
|
629
654
|
def diff_from(
|
|
630
655
|
self,
|
|
631
|
-
source: "
|
|
656
|
+
source: "Adapter",
|
|
632
657
|
diff_class: Type[Diff] = Diff,
|
|
633
658
|
flags: DiffSyncFlags = DiffSyncFlags.NONE,
|
|
634
659
|
callback: Optional[Callable[[StrType, int, int], None]] = None,
|
|
@@ -643,13 +668,17 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
643
668
|
calculation of the diff proceeds.
|
|
644
669
|
"""
|
|
645
670
|
differ = DiffSyncDiffer(
|
|
646
|
-
src_diffsync=source,
|
|
671
|
+
src_diffsync=source,
|
|
672
|
+
dst_diffsync=self,
|
|
673
|
+
flags=flags,
|
|
674
|
+
diff_class=diff_class,
|
|
675
|
+
callback=callback,
|
|
647
676
|
)
|
|
648
677
|
return differ.calculate_diffs()
|
|
649
678
|
|
|
650
679
|
def diff_to(
|
|
651
680
|
self,
|
|
652
|
-
target: "
|
|
681
|
+
target: "Adapter",
|
|
653
682
|
diff_class: Type[Diff] = Diff,
|
|
654
683
|
flags: DiffSyncFlags = DiffSyncFlags.NONE,
|
|
655
684
|
callback: Optional[Callable[[StrType, int, int], None]] = None,
|
|
@@ -678,7 +707,9 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
678
707
|
return self.store.get_all_model_names()
|
|
679
708
|
|
|
680
709
|
def get(
|
|
681
|
-
self,
|
|
710
|
+
self,
|
|
711
|
+
obj: Union[StrType, DiffSyncModel, Type[DiffSyncModel]],
|
|
712
|
+
identifier: Union[StrType, Dict],
|
|
682
713
|
) -> DiffSyncModel:
|
|
683
714
|
"""Get one object from the data store based on its unique id.
|
|
684
715
|
|
|
@@ -693,7 +724,9 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
693
724
|
return self.store.get(model=obj, identifier=identifier)
|
|
694
725
|
|
|
695
726
|
def get_or_none(
|
|
696
|
-
self,
|
|
727
|
+
self,
|
|
728
|
+
obj: Union[StrType, DiffSyncModel, Type[DiffSyncModel]],
|
|
729
|
+
identifier: Union[StrType, Dict],
|
|
697
730
|
) -> Optional[DiffSyncModel]:
|
|
698
731
|
"""Get one object from the data store based on its unique id or get a None
|
|
699
732
|
|
|
@@ -724,7 +757,9 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
724
757
|
return self.store.get_all(model=obj)
|
|
725
758
|
|
|
726
759
|
def get_by_uids(
|
|
727
|
-
self,
|
|
760
|
+
self,
|
|
761
|
+
uids: List[StrType],
|
|
762
|
+
obj: Union[StrType, DiffSyncModel, Type[DiffSyncModel]],
|
|
728
763
|
) -> List[DiffSyncModel]:
|
|
729
764
|
"""Get multiple objects from the store by their unique IDs/Keys and type.
|
|
730
765
|
|
|
@@ -859,5 +894,15 @@ class DiffSync: # pylint: disable=too-many-public-methods
|
|
|
859
894
|
return self.store.count(model=model)
|
|
860
895
|
|
|
861
896
|
|
|
862
|
-
|
|
863
|
-
|
|
897
|
+
def DiffSync(*args: Any, **kwargs: Any) -> Adapter: # noqa pylint: disable=invalid-name
|
|
898
|
+
"""For backwards-compatibility, keep around the old name."""
|
|
899
|
+
|
|
900
|
+
warnings.warn(
|
|
901
|
+
"'diffsync.DiffSync' is deprecated and will be removed with 2.1, use 'diffsync.Adapter' instead.",
|
|
902
|
+
DeprecationWarning,
|
|
903
|
+
)
|
|
904
|
+
return Adapter(*args, **kwargs)
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
# DiffSyncModel references Adapter and Adapter references DiffSyncModel. Break the typing loop:
|
|
908
|
+
DiffSyncModel.model_rebuild()
|
|
@@ -26,7 +26,7 @@ from .utils import intersection, symmetric_difference
|
|
|
26
26
|
|
|
27
27
|
if TYPE_CHECKING: # pragma: no cover
|
|
28
28
|
# For type annotation purposes, we have a circular import loop between __init__.py and this file.
|
|
29
|
-
from . import
|
|
29
|
+
from . import Adapter, DiffSyncModel # pylint: disable=cyclic-import
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
class DiffSyncDiffer: # pylint: disable=too-many-instance-attributes
|
|
@@ -37,8 +37,8 @@ class DiffSyncDiffer: # pylint: disable=too-many-instance-attributes
|
|
|
37
37
|
|
|
38
38
|
def __init__( # pylint: disable=too-many-arguments
|
|
39
39
|
self,
|
|
40
|
-
src_diffsync: "
|
|
41
|
-
dst_diffsync: "
|
|
40
|
+
src_diffsync: "Adapter",
|
|
41
|
+
dst_diffsync: "Adapter",
|
|
42
42
|
flags: DiffSyncFlags,
|
|
43
43
|
diff_class: Type[Diff] = Diff,
|
|
44
44
|
callback: Optional[Callable[[str, int, int], None]] = None,
|
|
@@ -288,8 +288,8 @@ class DiffSyncSyncer: # pylint: disable=too-many-instance-attributes
|
|
|
288
288
|
def __init__( # pylint: disable=too-many-arguments
|
|
289
289
|
self,
|
|
290
290
|
diff: Diff,
|
|
291
|
-
src_diffsync: "
|
|
292
|
-
dst_diffsync: "
|
|
291
|
+
src_diffsync: "Adapter",
|
|
292
|
+
dst_diffsync: "Adapter",
|
|
293
293
|
flags: DiffSyncFlags,
|
|
294
294
|
callback: Optional[Callable[[str, int, int], None]] = None,
|
|
295
295
|
):
|
|
@@ -425,7 +425,7 @@ class DiffSyncSyncer: # pylint: disable=too-many-instance-attributes
|
|
|
425
425
|
if self.action == DiffSyncActions.CREATE:
|
|
426
426
|
if dst_model is not None:
|
|
427
427
|
raise ObjectNotCreated(f"Failed to create {self.model_class.get_type()} {ids} - it already exists!")
|
|
428
|
-
dst_model = self.model_class.create(
|
|
428
|
+
dst_model = self.model_class.create(adapter=self.dst_diffsync, ids=ids, attrs=attrs)
|
|
429
429
|
elif self.action == DiffSyncActions.UPDATE:
|
|
430
430
|
if dst_model is None:
|
|
431
431
|
raise ObjectNotUpdated(f"Failed to update {self.model_class.get_type()} {ids} - not found!")
|
|
@@ -6,7 +6,7 @@ from diffsync.exceptions import ObjectNotFound
|
|
|
6
6
|
|
|
7
7
|
if TYPE_CHECKING:
|
|
8
8
|
from diffsync import DiffSyncModel
|
|
9
|
-
from diffsync import
|
|
9
|
+
from diffsync import Adapter
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class BaseStore:
|
|
@@ -15,12 +15,12 @@ class BaseStore:
|
|
|
15
15
|
def __init__(
|
|
16
16
|
self, # pylint: disable=unused-argument
|
|
17
17
|
*args: Any, # pylint: disable=unused-argument
|
|
18
|
-
|
|
18
|
+
adapter: Optional["Adapter"] = None,
|
|
19
19
|
name: str = "",
|
|
20
20
|
**kwargs: Any, # pylint: disable=unused-argument
|
|
21
21
|
) -> None:
|
|
22
22
|
"""Init method for BaseStore."""
|
|
23
|
-
self.
|
|
23
|
+
self.adapter = adapter
|
|
24
24
|
self.name = name or self.__class__.__name__
|
|
25
25
|
self._log = structlog.get_logger().new(store=self)
|
|
26
26
|
|
|
@@ -95,8 +95,8 @@ class BaseStore:
|
|
|
95
95
|
|
|
96
96
|
self.remove_item(modelname, uid)
|
|
97
97
|
|
|
98
|
-
if obj.
|
|
99
|
-
obj.
|
|
98
|
+
if obj.adapter:
|
|
99
|
+
obj.adapter = None
|
|
100
100
|
|
|
101
101
|
if remove_children:
|
|
102
102
|
for child_type, child_fieldname in obj.get_children_mapping().items():
|
|
@@ -243,9 +243,9 @@ class BaseStore:
|
|
|
243
243
|
"""Get object class and model name for a model."""
|
|
244
244
|
if isinstance(model, str):
|
|
245
245
|
modelname = model
|
|
246
|
-
if not hasattr(self.
|
|
246
|
+
if not hasattr(self.adapter, model):
|
|
247
247
|
return None, modelname
|
|
248
|
-
object_class = getattr(self.
|
|
248
|
+
object_class = getattr(self.adapter, model)
|
|
249
249
|
else:
|
|
250
250
|
object_class = model
|
|
251
251
|
modelname = model.get_type()
|
|
@@ -108,8 +108,8 @@ class LocalStore(BaseStore):
|
|
|
108
108
|
# Return so we don't have to change anything on the existing object and underlying data
|
|
109
109
|
return
|
|
110
110
|
|
|
111
|
-
if not obj.
|
|
112
|
-
obj.
|
|
111
|
+
if not obj.adapter:
|
|
112
|
+
obj.adapter = self.adapter
|
|
113
113
|
|
|
114
114
|
self._data[modelname][uid] = obj
|
|
115
115
|
|
|
@@ -65,7 +65,7 @@ class RedisStore(BaseStore):
|
|
|
65
65
|
pickled_object = self._store.get(key)
|
|
66
66
|
if pickled_object:
|
|
67
67
|
obj_result = loads(pickled_object) # nosec
|
|
68
|
-
obj_result.
|
|
68
|
+
obj_result.adapter = self.adapter
|
|
69
69
|
return obj_result
|
|
70
70
|
raise ObjectNotFound(f"{key} not present in Cache")
|
|
71
71
|
|
|
@@ -178,7 +178,7 @@ class RedisStore(BaseStore):
|
|
|
178
178
|
|
|
179
179
|
# Remove the diffsync object before sending to Redis
|
|
180
180
|
obj_copy = copy.copy(obj)
|
|
181
|
-
obj_copy.
|
|
181
|
+
obj_copy.adapter = None
|
|
182
182
|
|
|
183
183
|
self._store.set(object_key, dumps(obj_copy))
|
|
184
184
|
|
|
@@ -193,7 +193,7 @@ class RedisStore(BaseStore):
|
|
|
193
193
|
|
|
194
194
|
object_key = self._get_key_for_object(modelname, uid)
|
|
195
195
|
obj_copy = copy.copy(obj)
|
|
196
|
-
obj_copy.
|
|
196
|
+
obj_copy.adapter = None
|
|
197
197
|
|
|
198
198
|
self._store.set(object_key, dumps(obj_copy))
|
|
199
199
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "diffsync"
|
|
3
|
-
version = "
|
|
3
|
+
version = "v2.0.0"
|
|
4
4
|
description = "Library to easily sync/diff/update 2 different data sources"
|
|
5
5
|
authors = ["Network to Code, LLC <info@networktocode.com>"]
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -17,7 +17,7 @@ include = [
|
|
|
17
17
|
|
|
18
18
|
[tool.poetry.dependencies]
|
|
19
19
|
python = ">=3.8,<4.0"
|
|
20
|
-
pydantic = "^
|
|
20
|
+
pydantic = "^2.0.0"
|
|
21
21
|
structlog = ">= 20.1.0, < 23.0.0"
|
|
22
22
|
packaging = ">= 21.3, < 24.0"
|
|
23
23
|
colorama = {version = "^0.4.3", optional = true}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|