django-dato-sync 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_dato_sync-0.0.1/PKG-INFO +127 -0
- django_dato_sync-0.0.1/README.md +110 -0
- django_dato_sync-0.0.1/dato_sync/__init__.py +0 -0
- django_dato_sync-0.0.1/dato_sync/apps.py +11 -0
- django_dato_sync-0.0.1/dato_sync/datocms_api.py +36 -0
- django_dato_sync-0.0.1/dato_sync/decorators.py +21 -0
- django_dato_sync-0.0.1/dato_sync/errors.py +16 -0
- django_dato_sync-0.0.1/dato_sync/fetcher.py +64 -0
- django_dato_sync-0.0.1/dato_sync/models.py +59 -0
- django_dato_sync-0.0.1/dato_sync/sync_options.py +27 -0
- django_dato_sync-0.0.1/dato_sync/urls.py +8 -0
- django_dato_sync-0.0.1/dato_sync/util.py +30 -0
- django_dato_sync-0.0.1/dato_sync/views.py +23 -0
- django_dato_sync-0.0.1/django_dato_sync.egg-info/PKG-INFO +127 -0
- django_dato_sync-0.0.1/django_dato_sync.egg-info/SOURCES.txt +18 -0
- django_dato_sync-0.0.1/django_dato_sync.egg-info/dependency_links.txt +1 -0
- django_dato_sync-0.0.1/django_dato_sync.egg-info/top_level.txt +1 -0
- django_dato_sync-0.0.1/pyproject.toml +23 -0
- django_dato_sync-0.0.1/setup.cfg +4 -0
- django_dato_sync-0.0.1/setup.py +35 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: django-dato-sync
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: A sync utility for storing DatoCMS objects in django.
|
|
5
|
+
Home-page: https://www.dreipol.ch/
|
|
6
|
+
Author: dreipol AG
|
|
7
|
+
Author-email: Laila Becker <laila.becker@dreipol.ch>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Project-URL: Homepage, https://github.com/dreipol/django-dato-sync
|
|
10
|
+
Project-URL: Issues, https://github.com/dreipol/django-dato-sync/issues
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Requires-Python: >=3.12
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
Dynamic: author
|
|
16
|
+
Dynamic: home-page
|
|
17
|
+
|
|
18
|
+
# django-dato-sync
|
|
19
|
+
|
|
20
|
+
django-dato-sync enables you to easily sync Dato records into your django database. Features include:
|
|
21
|
+
|
|
22
|
+
- Delta sync
|
|
23
|
+
- Automatic sync via Dato webhooks
|
|
24
|
+
- Localization support with [django-modeltranslation](https://github.com/deschler/django-modeltranslation)
|
|
25
|
+
- Configuration of which fields to sync
|
|
26
|
+
- Collecting information of multiple Dato records into a single django object
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
|
|
30
|
+
1. Install the pip package:
|
|
31
|
+
```shell
|
|
32
|
+
pipenv install django-dato-sync
|
|
33
|
+
```
|
|
34
|
+
2. Add to your installed apps:
|
|
35
|
+
```py
|
|
36
|
+
INSTALLED_APPS = [
|
|
37
|
+
"django.contrib.admin",
|
|
38
|
+
"django.contrib.auth",
|
|
39
|
+
"django.contrib.staticfiles",
|
|
40
|
+
...
|
|
41
|
+
"dato_sync",
|
|
42
|
+
]
|
|
43
|
+
```
|
|
44
|
+
3. Setup at least the following settings:
|
|
45
|
+
```py
|
|
46
|
+
DATOCMS_API_TOKEN: str = ...
|
|
47
|
+
DATOCMS_API_URL: str = ...
|
|
48
|
+
DATOCMS_ENVIRONMENT: str = ...
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Optional: Setup for automatic syncing via Webhooks
|
|
52
|
+
1. Add the following to your `urls.py`:
|
|
53
|
+
```py
|
|
54
|
+
urlpatterns = [
|
|
55
|
+
...
|
|
56
|
+
path("", include("dato_sync.urls"))
|
|
57
|
+
]
|
|
58
|
+
```
|
|
59
|
+
2. Setup the authentication header Dato will use in your `settings.py`
|
|
60
|
+
```py
|
|
61
|
+
DATO_SYNC_WEBHOOK_EXPECTED_AUTH: str = "Basic ..."
|
|
62
|
+
```
|
|
63
|
+
3. In Dato navigate to Project Setting > Automations > Webhooks and click "Add a new webhook"
|
|
64
|
+
4. Configure the webhook to trigger on "Publish", "Unpublish", and "Delete" for any records you want to sync to django
|
|
65
|
+
5. Specify the URL as `https://<your-django-server-address>/dato-sync/sync/`
|
|
66
|
+
6. Configure the "HTTP basic auth" to match the header you configured in step 2
|
|
67
|
+
|
|
68
|
+
## Usage
|
|
69
|
+
1. Create your local django model but make sure it inherits from `DatoModel` (it will automatically have fields for the `dato_identifier` (pk), `created` and `modified` dates (corresponding to changes made in Dato) and a `deleted` field indicating a soft delete)
|
|
70
|
+
```py
|
|
71
|
+
class MyModel(DatoModel):
|
|
72
|
+
name = models.TextField()
|
|
73
|
+
order = models.IntegerField()
|
|
74
|
+
note = models.TextField()
|
|
75
|
+
```
|
|
76
|
+
2. Create a `dato_sync.py` file in your app:
|
|
77
|
+
```py
|
|
78
|
+
@fetch_from_dato(MyModel)
|
|
79
|
+
class MyModelSyncOptions(SyncOptions):
|
|
80
|
+
dato_model_path = "my_model"
|
|
81
|
+
field_mappings = [
|
|
82
|
+
"order" |position_in_parent,
|
|
83
|
+
"name",
|
|
84
|
+
]
|
|
85
|
+
```
|
|
86
|
+
3. To sync either have Dato call the webhook (see above) or use
|
|
87
|
+
```shell
|
|
88
|
+
python manage.py sync_dato [--force-full-sync]
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Configuration Options
|
|
92
|
+
#### dato_model_path
|
|
93
|
+
Configure the dato entity your model corresponds to. If you are mapping a Dato model directly this is just its model id. If you're using blocks you can chain them like `my_model.parent_block.child_block`.
|
|
94
|
+
|
|
95
|
+
⚠️ Make sure that all blocks have the same schema. Fields that can contain different types of blocks are currently not supported.
|
|
96
|
+
|
|
97
|
+
#### field_mappings
|
|
98
|
+
Here you specify which fields should be synced with dato. If you leave out fields (like the `notes` field in the example above they can be edited locally).
|
|
99
|
+
|
|
100
|
+
In the simplest case when the name of your field in django corresponds to the api name in Dato, you can simply add it to the field mappings as we did with `"name"`. For more complicated scenarios you can write:
|
|
101
|
+
```py
|
|
102
|
+
field_mappings = [
|
|
103
|
+
"name" |from_dato_path("my_model.title", localized=True, absolute=True),
|
|
104
|
+
]
|
|
105
|
+
```
|
|
106
|
+
This allows you to
|
|
107
|
+
|
|
108
|
+
- specify a different name / path to take the value from
|
|
109
|
+
- `localized` allows you to fetch localizations from Dato and store them using django-modeltranslation
|
|
110
|
+
- `absolute` allows you to access properties of the parent entities by specifying the field to take the value from starting from the top of the Dato query rather than the path specified in `dato_model_path`
|
|
111
|
+
|
|
112
|
+
Additonally the following are also available:
|
|
113
|
+
- `|position_in_parent` to obtain the position of the item in its parent
|
|
114
|
+
- `|flattened_position` to obtain a global order by flattening the list across all paths
|
|
115
|
+
|
|
116
|
+
#### ArrayFields
|
|
117
|
+
Postgres ArrayFields are supported ⚠️ so long as there is no nesting on either the Dato or django side ⚠️. Simply specify the path like:
|
|
118
|
+
```py
|
|
119
|
+
"tags" |from_dato_path("tags.name")
|
|
120
|
+
```
|
|
121
|
+
and django-dato-sync will automatically collect the names of all tags into an array.
|
|
122
|
+
|
|
123
|
+
## Tips and Tricks
|
|
124
|
+
- Foreign key relationships are not supported directly, but you can use django's `..._id` field to set the id of another Dato object
|
|
125
|
+
- ⚠️ Make sure to sync the related objects first to avoid foreign key constraint violations. Sync operations are executed in the same order they appear in the `dato_sync.py` file.
|
|
126
|
+
- For one-to-many relationships use absolute paths to access the parent's id
|
|
127
|
+
- You can create multiple sync jobs for the same django model to collect all instances of a block across multiple models into one django table
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# django-dato-sync
|
|
2
|
+
|
|
3
|
+
django-dato-sync enables you to easily sync Dato records into your django database. Features include:
|
|
4
|
+
|
|
5
|
+
- Delta sync
|
|
6
|
+
- Automatic sync via Dato webhooks
|
|
7
|
+
- Localization support with [django-modeltranslation](https://github.com/deschler/django-modeltranslation)
|
|
8
|
+
- Configuration of which fields to sync
|
|
9
|
+
- Collecting information of multiple Dato records into a single django object
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
1. Install the pip package:
|
|
14
|
+
```shell
|
|
15
|
+
pipenv install django-dato-sync
|
|
16
|
+
```
|
|
17
|
+
2. Add to your installed apps:
|
|
18
|
+
```py
|
|
19
|
+
INSTALLED_APPS = [
|
|
20
|
+
"django.contrib.admin",
|
|
21
|
+
"django.contrib.auth",
|
|
22
|
+
"django.contrib.staticfiles",
|
|
23
|
+
...
|
|
24
|
+
"dato_sync",
|
|
25
|
+
]
|
|
26
|
+
```
|
|
27
|
+
3. Setup at least the following settings:
|
|
28
|
+
```py
|
|
29
|
+
DATOCMS_API_TOKEN: str = ...
|
|
30
|
+
DATOCMS_API_URL: str = ...
|
|
31
|
+
DATOCMS_ENVIRONMENT: str = ...
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### Optional: Setup for automatic syncing via Webhooks
|
|
35
|
+
1. Add the following to your `urls.py`:
|
|
36
|
+
```py
|
|
37
|
+
urlpatterns = [
|
|
38
|
+
...
|
|
39
|
+
path("", include("dato_sync.urls"))
|
|
40
|
+
]
|
|
41
|
+
```
|
|
42
|
+
2. Setup the authentication header Dato will use in your `settings.py`
|
|
43
|
+
```py
|
|
44
|
+
DATO_SYNC_WEBHOOK_EXPECTED_AUTH: str = "Basic ..."
|
|
45
|
+
```
|
|
46
|
+
3. In Dato navigate to Project Setting > Automations > Webhooks and click "Add a new webhook"
|
|
47
|
+
4. Configure the webhook to trigger on "Publish", "Unpublish", and "Delete" for any records you want to sync to django
|
|
48
|
+
5. Specify the URL as `https://<your-django-server-address>/dato-sync/sync/`
|
|
49
|
+
6. Configure the "HTTP basic auth" to match the header you configured in step 2
|
|
50
|
+
|
|
51
|
+
## Usage
|
|
52
|
+
1. Create your local django model but make sure it inherits from `DatoModel` (it will automatically have fields for the `dato_identifier` (pk), `created` and `modified` dates (corresponding to changes made in Dato) and a `deleted` field indicating a soft delete)
|
|
53
|
+
```py
|
|
54
|
+
class MyModel(DatoModel):
|
|
55
|
+
name = models.TextField()
|
|
56
|
+
order = models.IntegerField()
|
|
57
|
+
note = models.TextField()
|
|
58
|
+
```
|
|
59
|
+
2. Create a `dato_sync.py` file in your app:
|
|
60
|
+
```py
|
|
61
|
+
@fetch_from_dato(MyModel)
|
|
62
|
+
class MyModelSyncOptions(SyncOptions):
|
|
63
|
+
dato_model_path = "my_model"
|
|
64
|
+
field_mappings = [
|
|
65
|
+
"order" |position_in_parent,
|
|
66
|
+
"name",
|
|
67
|
+
]
|
|
68
|
+
```
|
|
69
|
+
3. To sync either have Dato call the webhook (see above) or use
|
|
70
|
+
```shell
|
|
71
|
+
python manage.py sync_dato [--force-full-sync]
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Configuration Options
|
|
75
|
+
#### dato_model_path
|
|
76
|
+
Configure the dato entity your model corresponds to. If you are mapping a Dato model directly this is just its model id. If you're using blocks you can chain them like `my_model.parent_block.child_block`.
|
|
77
|
+
|
|
78
|
+
⚠️ Make sure that all blocks have the same schema. Fields that can contain different types of blocks are currently not supported.
|
|
79
|
+
|
|
80
|
+
#### field_mappings
|
|
81
|
+
Here you specify which fields should be synced with dato. If you leave out fields (like the `notes` field in the example above they can be edited locally).
|
|
82
|
+
|
|
83
|
+
In the simplest case when the name of your field in django corresponds to the api name in Dato, you can simply add it to the field mappings as we did with `"name"`. For more complicated scenarios you can write:
|
|
84
|
+
```py
|
|
85
|
+
field_mappings = [
|
|
86
|
+
"name" |from_dato_path("my_model.title", localized=True, absolute=True),
|
|
87
|
+
]
|
|
88
|
+
```
|
|
89
|
+
This allows you to
|
|
90
|
+
|
|
91
|
+
- specify a different name / path to take the value from
|
|
92
|
+
- `localized` allows you to fetch localizations from Dato and store them using django-modeltranslation
|
|
93
|
+
- `absolute` allows you to access properties of the parent entities by specifying the field to take the value from starting from the top of the Dato query rather than the path specified in `dato_model_path`
|
|
94
|
+
|
|
95
|
+
Additonally the following are also available:
|
|
96
|
+
- `|position_in_parent` to obtain the position of the item in its parent
|
|
97
|
+
- `|flattened_position` to obtain a global order by flattening the list across all paths
|
|
98
|
+
|
|
99
|
+
#### ArrayFields
|
|
100
|
+
Postgres ArrayFields are supported ⚠️ so long as there is no nesting on either the Dato or django side ⚠️. Simply specify the path like:
|
|
101
|
+
```py
|
|
102
|
+
"tags" |from_dato_path("tags.name")
|
|
103
|
+
```
|
|
104
|
+
and django-dato-sync will automatically collect the names of all tags into an array.
|
|
105
|
+
|
|
106
|
+
## Tips and Tricks
|
|
107
|
+
- Foreign key relationships are not supported directly, but you can use django's `..._id` field to set the id of another Dato object
|
|
108
|
+
- ⚠️ Make sure to sync the related objects first to avoid foreign key constraint violations. Sync operations are executed in the same order they appear in the `dato_sync.py` file.
|
|
109
|
+
- For one-to-many relationships use absolute paths to access the parent's id
|
|
110
|
+
- You can create multiple sync jobs for the same django model to collect all instances of a block across multiple models into one django table
|
|
File without changes
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from django.conf import settings
|
|
2
|
+
import requests
|
|
3
|
+
from requests.auth import AuthBase
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DatoTokenAuth(AuthBase):
|
|
7
|
+
"""Implements a token authentication scheme."""
|
|
8
|
+
|
|
9
|
+
def __init__(self, token, environment):
|
|
10
|
+
self.token = token
|
|
11
|
+
self.environment = environment
|
|
12
|
+
|
|
13
|
+
def __call__(self, request):
|
|
14
|
+
"""Attach an API token to the Authorization header."""
|
|
15
|
+
request.headers["Authorization"] = f"Bearer {self.token}"
|
|
16
|
+
request.headers["Content-Type"] = "application/json"
|
|
17
|
+
request.headers["Accept"] = "application/json"
|
|
18
|
+
request.headers["X-Environment"] = self.environment
|
|
19
|
+
return request
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def fetch_datocms_content(language: str, query: str) -> dict:
|
|
23
|
+
api_token = settings.DATOCMS_API_TOKEN
|
|
24
|
+
api_url = f"{settings.DATOCMS_API_URL}/"
|
|
25
|
+
environment = settings.DATOCMS_ENVIRONMENT
|
|
26
|
+
variables = {"locale": language}
|
|
27
|
+
response = requests.post(
|
|
28
|
+
api_url, auth=DatoTokenAuth(api_token, environment), json={"query": query, "variables": variables}
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
if response.status_code == 200:
|
|
32
|
+
data = response.json()
|
|
33
|
+
return data.get("data")
|
|
34
|
+
else:
|
|
35
|
+
print(f"Error: {response.status_code} - {response.text}")
|
|
36
|
+
return {}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from typing import Type, TypeVar, Callable
|
|
2
|
+
|
|
3
|
+
from dato_sync.fetcher import fetcher
|
|
4
|
+
from dato_sync.models import DatoModel
|
|
5
|
+
from dato_sync.sync_options import SyncOptions
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
_OptionsTypeT = TypeVar("_OptionsTypeT", bound=type[SyncOptions])
|
|
9
|
+
|
|
10
|
+
def fetch_from_dato(model_class: Type[DatoModel]) -> Callable[[_OptionsTypeT], _OptionsTypeT]:
|
|
11
|
+
"""
|
|
12
|
+
Populate the model with objects managed in the dato CMS.
|
|
13
|
+
"""
|
|
14
|
+
def wrapper(opts_class: _OptionsTypeT) -> _OptionsTypeT:
|
|
15
|
+
if not issubclass(opts_class, SyncOptions):
|
|
16
|
+
raise ValueError("Wrapped class must subclass SyncOptions.")
|
|
17
|
+
|
|
18
|
+
fetcher.register(model_class, opts_class)
|
|
19
|
+
return opts_class
|
|
20
|
+
|
|
21
|
+
return wrapper
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
class BadConfigurationError(Exception):
|
|
2
|
+
def __init__(self, message: str):
|
|
3
|
+
self.message = message
|
|
4
|
+
|
|
5
|
+
def __str__(self):
|
|
6
|
+
return f"Bad configuration of the dato_sync plugin: {self.message}"
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class IllegalSyncOptionsError(Exception):
|
|
10
|
+
def __init__(self, model: object, options_type: object, message: str):
|
|
11
|
+
self.model = model
|
|
12
|
+
self.options_type = options_type
|
|
13
|
+
self.message = message
|
|
14
|
+
|
|
15
|
+
def __str__(self):
|
|
16
|
+
return f"The sync options provided for {self.model} in {self.options_type} are illegal: {self.message}"
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
from typing import Type
|
|
2
|
+
|
|
3
|
+
from django.conf import settings
|
|
4
|
+
from django.db.models import Max
|
|
5
|
+
|
|
6
|
+
from dato_sync.datocms_api import fetch_datocms_content
|
|
7
|
+
from dato_sync.query_tree import QueryTree, QueryGenerator, ResponseParser
|
|
8
|
+
from dato_sync.sync_options import SyncOptions, DatoFieldPath
|
|
9
|
+
from search.config import DatoModel
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Fetcher:
|
|
13
|
+
def __init__(self):
|
|
14
|
+
self.jobs: list[SyncOptions] = []
|
|
15
|
+
|
|
16
|
+
def register(self, model: Type[DatoModel], options: SyncOptions):
|
|
17
|
+
options.django_model = model
|
|
18
|
+
self.jobs.append(options)
|
|
19
|
+
|
|
20
|
+
def fetch(self, force_full_sync: bool):
|
|
21
|
+
default_locale = settings.LANGUAGE_CODE
|
|
22
|
+
|
|
23
|
+
seen_ids = dict()
|
|
24
|
+
|
|
25
|
+
for job in self.jobs:
|
|
26
|
+
sanitized_mappings = [
|
|
27
|
+
mapping if isinstance(mapping, DatoFieldPath) else DatoFieldPath(mapping)
|
|
28
|
+
for mapping in job.field_mappings]
|
|
29
|
+
|
|
30
|
+
if force_full_sync:
|
|
31
|
+
min_date = None
|
|
32
|
+
else:
|
|
33
|
+
min_date = job.django_model.objects.aggregate(max_date=Max("modified"))["max_date"]
|
|
34
|
+
|
|
35
|
+
query_tree = QueryTree(
|
|
36
|
+
job=job,
|
|
37
|
+
min_date=min_date,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
for mapping in sanitized_mappings:
|
|
41
|
+
query_tree.insert_mapping(mapping, job)
|
|
42
|
+
|
|
43
|
+
base_query = QueryGenerator(for_localization=False).generate_query(query_tree)
|
|
44
|
+
|
|
45
|
+
response = fetch_datocms_content(default_locale, base_query)
|
|
46
|
+
if any(mapping.is_localized for mapping in sanitized_mappings):
|
|
47
|
+
localization_query = QueryGenerator(for_localization=True).generate_query(query_tree)
|
|
48
|
+
localization_responses = {language: fetch_datocms_content(language, localization_query)
|
|
49
|
+
for language, _ in settings.LANGUAGES
|
|
50
|
+
if language != default_locale}
|
|
51
|
+
else:
|
|
52
|
+
localization_responses = dict()
|
|
53
|
+
|
|
54
|
+
new_ids = ResponseParser(job).parse_response(response, localization_responses, query_tree)
|
|
55
|
+
ids_set: set[str] = seen_ids.get(job.django_model, set())
|
|
56
|
+
ids_set = ids_set.union(new_ids)
|
|
57
|
+
seen_ids[job.django_model] = ids_set
|
|
58
|
+
|
|
59
|
+
for model, ids_set in seen_ids.items():
|
|
60
|
+
(model.objects
|
|
61
|
+
.exclude(dato_identifier__in=ids_set)
|
|
62
|
+
.update(deleted=True))
|
|
63
|
+
|
|
64
|
+
fetcher = Fetcher()
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from django.db import models
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class DatoModel(models.Model):
|
|
5
|
+
dato_identifier = models.TextField(
|
|
6
|
+
primary_key=True,
|
|
7
|
+
blank=False,
|
|
8
|
+
null=False,
|
|
9
|
+
max_length=255
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
created = models.DateTimeField(
|
|
13
|
+
verbose_name="created",
|
|
14
|
+
db_index=True,
|
|
15
|
+
)
|
|
16
|
+
modified = models.DateTimeField(
|
|
17
|
+
verbose_name="modified",
|
|
18
|
+
db_index=True,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
deleted = models.BooleanField(default=False)
|
|
22
|
+
|
|
23
|
+
class Meta:
|
|
24
|
+
abstract = True
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def handle_dato_sync_registrations():
|
|
28
|
+
"""
|
|
29
|
+
Auto-discover INSTALLED_APPS translation.py modules and fail silently when
|
|
30
|
+
not present. This forces an import on them to register.
|
|
31
|
+
Also import explicit modules.
|
|
32
|
+
"""
|
|
33
|
+
import copy
|
|
34
|
+
from importlib import import_module
|
|
35
|
+
|
|
36
|
+
from django.apps import apps
|
|
37
|
+
from django.utils.module_loading import module_has_submodule
|
|
38
|
+
|
|
39
|
+
from dato_sync.fetcher import fetcher
|
|
40
|
+
|
|
41
|
+
mods = [(app_config.name, app_config.module) for app_config in apps.get_app_configs()]
|
|
42
|
+
|
|
43
|
+
for app, mod in mods:
|
|
44
|
+
# Attempt to import the app's dato_sync module.
|
|
45
|
+
module = "%s.dato_sync" % app
|
|
46
|
+
before_import_jobs = copy.copy(fetcher.jobs)
|
|
47
|
+
try:
|
|
48
|
+
import_module(module)
|
|
49
|
+
except ImportError:
|
|
50
|
+
# Reset the model registry to the state before the last import as
|
|
51
|
+
# this import will have to reoccur on the next request and this
|
|
52
|
+
# could raise NotRegistered and AlreadyRegistered exceptions
|
|
53
|
+
fetcher.jobs = before_import_jobs
|
|
54
|
+
|
|
55
|
+
# Decide whether to bubble up this error. If the app just
|
|
56
|
+
# doesn't have a dato_sync module, we can ignore the error
|
|
57
|
+
# attempting to import it, otherwise we want it to bubble up.
|
|
58
|
+
if module_has_submodule(mod, "dato_sync"):
|
|
59
|
+
raise
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class DatoFieldPath:
|
|
5
|
+
def __init__(
|
|
6
|
+
self,
|
|
7
|
+
django_field_name: str,
|
|
8
|
+
path: str | None = None,
|
|
9
|
+
is_localized: bool = False,
|
|
10
|
+
is_absolute: bool = False,
|
|
11
|
+
):
|
|
12
|
+
self.django_field_name = django_field_name
|
|
13
|
+
self.path = path or django_field_name
|
|
14
|
+
self.is_localized = is_localized
|
|
15
|
+
self.is_absolute = is_absolute
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SyncOptions(ABC):
|
|
19
|
+
@property
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def dato_model_path(self) -> str:
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
@abstractmethod
|
|
26
|
+
def field_mappings(self) -> list[str | DatoFieldPath]:
|
|
27
|
+
pass
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from typing import TypeVar, Callable, Generic
|
|
2
|
+
|
|
3
|
+
from dato_sync.sync_options import DatoFieldPath
|
|
4
|
+
|
|
5
|
+
T = TypeVar('T')
|
|
6
|
+
R = TypeVar('R')
|
|
7
|
+
|
|
8
|
+
_order_tag = '#order#'
|
|
9
|
+
_flattened_order_tag = '#flattened_order#'
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Suffix(Generic[T, R]):
|
|
13
|
+
def __init__(self, function: Callable[[T], R]):
|
|
14
|
+
self.function = function
|
|
15
|
+
|
|
16
|
+
def __ror__(self, other):
|
|
17
|
+
return self.function(other)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def from_dato_path(path: str | None = None, localized: bool = False, absolute: bool = False) -> Suffix[str, DatoFieldPath]:
|
|
21
|
+
return Suffix(lambda field: DatoFieldPath(field, path, localized, absolute))
|
|
22
|
+
|
|
23
|
+
position_in_parent = Suffix(lambda field: DatoFieldPath(django_field_name=field, path=_order_tag))
|
|
24
|
+
flattened_position = Suffix(lambda field: DatoFieldPath(django_field_name=field, path=_flattened_order_tag))
|
|
25
|
+
|
|
26
|
+
def to_camel_case(snake_str):
|
|
27
|
+
if snake_str.startswith('_'):
|
|
28
|
+
return snake_str
|
|
29
|
+
camel_string = "".join(x[0].upper() + x[1:] for x in snake_str.split("_"))
|
|
30
|
+
return snake_str[0].lower() + camel_string[1:]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from django.conf import settings
|
|
2
|
+
from django.core.handlers.wsgi import WSGIRequest
|
|
3
|
+
from django.http import HttpResponse
|
|
4
|
+
from django.views.decorators.csrf import csrf_exempt
|
|
5
|
+
from django.views.decorators.http import require_POST
|
|
6
|
+
|
|
7
|
+
from dato_sync.errors import BadConfigurationError
|
|
8
|
+
from dato_sync.fetcher import fetcher
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@csrf_exempt
|
|
12
|
+
@require_POST
|
|
13
|
+
def sync(request: WSGIRequest) -> HttpResponse:
|
|
14
|
+
"""Called by dato hook to initiate a delta sync"""
|
|
15
|
+
if not settings.DATO_SYNC_WEBHOOK_EXPECTED_AUTH:
|
|
16
|
+
raise BadConfigurationError("DATO_SYNC_WEBHOOK_EXPECTED_AUTH not configured")
|
|
17
|
+
|
|
18
|
+
if request.headers.get("Authorization") != settings.DATO_SYNC_WEBHOOK_EXPECTED_AUTH:
|
|
19
|
+
return HttpResponse("Unauthorized", status=401)
|
|
20
|
+
|
|
21
|
+
# TODO: use data from body
|
|
22
|
+
fetcher.fetch(force_full_sync=False)
|
|
23
|
+
return HttpResponse(status=204)
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: django-dato-sync
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: A sync utility for storing DatoCMS objects in django.
|
|
5
|
+
Home-page: https://www.dreipol.ch/
|
|
6
|
+
Author: dreipol AG
|
|
7
|
+
Author-email: Laila Becker <laila.becker@dreipol.ch>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Project-URL: Homepage, https://github.com/dreipol/django-dato-sync
|
|
10
|
+
Project-URL: Issues, https://github.com/dreipol/django-dato-sync/issues
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Requires-Python: >=3.12
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
Dynamic: author
|
|
16
|
+
Dynamic: home-page
|
|
17
|
+
|
|
18
|
+
# django-dato-sync
|
|
19
|
+
|
|
20
|
+
django-dato-sync enables you to easily sync Dato records into your django database. Features include:
|
|
21
|
+
|
|
22
|
+
- Delta sync
|
|
23
|
+
- Automatic sync via Dato webhooks
|
|
24
|
+
- Localization support with [django-modeltranslation](https://github.com/deschler/django-modeltranslation)
|
|
25
|
+
- Configuration of which fields to sync
|
|
26
|
+
- Collecting information of multiple Dato records into a single django object
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
|
|
30
|
+
1. Install the pip package:
|
|
31
|
+
```shell
|
|
32
|
+
pipenv install django-dato-sync
|
|
33
|
+
```
|
|
34
|
+
2. Add to your installed apps:
|
|
35
|
+
```py
|
|
36
|
+
INSTALLED_APPS = [
|
|
37
|
+
"django.contrib.admin",
|
|
38
|
+
"django.contrib.auth",
|
|
39
|
+
"django.contrib.staticfiles",
|
|
40
|
+
...
|
|
41
|
+
"dato_sync",
|
|
42
|
+
]
|
|
43
|
+
```
|
|
44
|
+
3. Setup at least the following settings:
|
|
45
|
+
```py
|
|
46
|
+
DATOCMS_API_TOKEN: str = ...
|
|
47
|
+
DATOCMS_API_URL: str = ...
|
|
48
|
+
DATOCMS_ENVIRONMENT: str = ...
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Optional: Setup for automatic syncing via Webhooks
|
|
52
|
+
1. Add the following to your `urls.py`:
|
|
53
|
+
```py
|
|
54
|
+
urlpatterns = [
|
|
55
|
+
...
|
|
56
|
+
path("", include("dato_sync.urls"))
|
|
57
|
+
]
|
|
58
|
+
```
|
|
59
|
+
2. Setup the authentication header Dato will use in your `settings.py`
|
|
60
|
+
```py
|
|
61
|
+
DATO_SYNC_WEBHOOK_EXPECTED_AUTH: str = "Basic ..."
|
|
62
|
+
```
|
|
63
|
+
3. In Dato navigate to Project Setting > Automations > Webhooks and click "Add a new webhook"
|
|
64
|
+
4. Configure the webhook to trigger on "Publish", "Unpublish", and "Delete" for any records you want to sync to django
|
|
65
|
+
5. Specify the URL as `https://<your-django-server-address>/dato-sync/sync/`
|
|
66
|
+
6. Configure the "HTTP basic auth" to match the header you configured in step 2
|
|
67
|
+
|
|
68
|
+
## Usage
|
|
69
|
+
1. Create your local django model but make sure it inherits from `DatoModel` (it will automatically have fields for the `dato_identifier` (pk), `created` and `modified` dates (corresponding to changes made in Dato) and a `deleted` field indicating a soft delete)
|
|
70
|
+
```py
|
|
71
|
+
class MyModel(DatoModel):
|
|
72
|
+
name = models.TextField()
|
|
73
|
+
order = models.IntegerField()
|
|
74
|
+
note = models.TextField()
|
|
75
|
+
```
|
|
76
|
+
2. Create a `dato_sync.py` file in your app:
|
|
77
|
+
```py
|
|
78
|
+
@fetch_from_dato(MyModel)
|
|
79
|
+
class MyModelSyncOptions(SyncOptions):
|
|
80
|
+
dato_model_path = "my_model"
|
|
81
|
+
field_mappings = [
|
|
82
|
+
"order" |position_in_parent,
|
|
83
|
+
"name",
|
|
84
|
+
]
|
|
85
|
+
```
|
|
86
|
+
3. To sync either have Dato call the webhook (see above) or use
|
|
87
|
+
```shell
|
|
88
|
+
python manage.py sync_dato [--force-full-sync]
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Configuration Options
|
|
92
|
+
#### dato_model_path
|
|
93
|
+
Configure the dato entity your model corresponds to. If you are mapping a Dato model directly this is just its model id. If you're using blocks you can chain them like `my_model.parent_block.child_block`.
|
|
94
|
+
|
|
95
|
+
⚠️ Make sure that all blocks have the same schema. Fields that can contain different types of blocks are currently not supported.
|
|
96
|
+
|
|
97
|
+
#### field_mappings
|
|
98
|
+
Here you specify which fields should be synced with dato. If you leave out fields (like the `notes` field in the example above they can be edited locally).
|
|
99
|
+
|
|
100
|
+
In the simplest case when the name of your field in django corresponds to the api name in Dato, you can simply add it to the field mappings as we did with `"name"`. For more complicated scenarios you can write:
|
|
101
|
+
```py
|
|
102
|
+
field_mappings = [
|
|
103
|
+
"name" |from_dato_path("my_model.title", localized=True, absolute=True),
|
|
104
|
+
]
|
|
105
|
+
```
|
|
106
|
+
This allows you to
|
|
107
|
+
|
|
108
|
+
- specify a different name / path to take the value from
|
|
109
|
+
- `localized` allows you to fetch localizations from Dato and store them using django-modeltranslation
|
|
110
|
+
- `absolute` allows you to access properties of the parent entities by specifying the field to take the value from starting from the top of the Dato query rather than the path specified in `dato_model_path`
|
|
111
|
+
|
|
112
|
+
Additonally the following are also available:
|
|
113
|
+
- `|position_in_parent` to obtain the position of the item in its parent
|
|
114
|
+
- `|flattened_position` to obtain a global order by flattening the list across all paths
|
|
115
|
+
|
|
116
|
+
#### ArrayFields
|
|
117
|
+
Postgres ArrayFields are supported ⚠️ so long as there is no nesting on either the Dato or django side ⚠️. Simply specify the path like:
|
|
118
|
+
```py
|
|
119
|
+
"tags" |from_dato_path("tags.name")
|
|
120
|
+
```
|
|
121
|
+
and django-dato-sync will automatically collect the names of all tags into an array.
|
|
122
|
+
|
|
123
|
+
## Tips and Tricks
|
|
124
|
+
- Foreign key relationships are not supported directly, but you can use django's `..._id` field to set the id of another Dato object
|
|
125
|
+
- ⚠️ Make sure to sync the related objects first to avoid foreign key constraint violations. Sync operations are executed in the same order they appear in the `dato_sync.py` file.
|
|
126
|
+
- For one-to-many relationships use absolute paths to access the parent's id
|
|
127
|
+
- You can create multiple sync jobs for the same django model to collect all instances of a block across multiple models into one django table
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
setup.py
|
|
4
|
+
dato_sync/__init__.py
|
|
5
|
+
dato_sync/apps.py
|
|
6
|
+
dato_sync/datocms_api.py
|
|
7
|
+
dato_sync/decorators.py
|
|
8
|
+
dato_sync/errors.py
|
|
9
|
+
dato_sync/fetcher.py
|
|
10
|
+
dato_sync/models.py
|
|
11
|
+
dato_sync/sync_options.py
|
|
12
|
+
dato_sync/urls.py
|
|
13
|
+
dato_sync/util.py
|
|
14
|
+
dato_sync/views.py
|
|
15
|
+
django_dato_sync.egg-info/PKG-INFO
|
|
16
|
+
django_dato_sync.egg-info/SOURCES.txt
|
|
17
|
+
django_dato_sync.egg-info/dependency_links.txt
|
|
18
|
+
django_dato_sync.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dato_sync
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools >= 77.0.3"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "django-dato-sync"
|
|
7
|
+
version = "0.0.1"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name="Laila Becker", email="laila.becker@dreipol.ch" },
|
|
10
|
+
]
|
|
11
|
+
description = "A sync utility for storing DatoCMS objects in django."
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
requires-python = ">=3.12"
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Operating System :: OS Independent",
|
|
17
|
+
]
|
|
18
|
+
license = "MIT"
|
|
19
|
+
license-files = ["LICEN[CS]E*"]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
Homepage = "https://github.com/dreipol/django-dato-sync"
|
|
23
|
+
Issues = "https://github.com/dreipol/django-dato-sync/issues"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
from setuptools import setup
|
|
4
|
+
|
|
5
|
+
README = open(os.path.join(os.path.dirname(__file__), 'README.md')).read()
|
|
6
|
+
|
|
7
|
+
reqs = ['Django>=5.0']
|
|
8
|
+
|
|
9
|
+
# allow setup.py to be run from any path
|
|
10
|
+
os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))
|
|
11
|
+
|
|
12
|
+
setup(
|
|
13
|
+
name='django-dato-sync',
|
|
14
|
+
version='1.0.0',
|
|
15
|
+
packages=['dato_sync'],
|
|
16
|
+
include_package_data=True,
|
|
17
|
+
license='MIT License',
|
|
18
|
+
description='django-dato-sync lets you import data from DatoCMS into the django DB',
|
|
19
|
+
long_description=README,
|
|
20
|
+
url='https://www.dreipol.ch/',
|
|
21
|
+
author='dreipol AG',
|
|
22
|
+
author_email='dev@dreipol.ch',
|
|
23
|
+
classifiers=[
|
|
24
|
+
'Environment :: Web Environment',
|
|
25
|
+
'Framework :: Django',
|
|
26
|
+
'Intended Audience :: Developers',
|
|
27
|
+
'License :: OSI Approved :: MIT License', # example license
|
|
28
|
+
'Operating System :: OS Independent',
|
|
29
|
+
'Programming Language :: Python',
|
|
30
|
+
'Programming Language :: Python :: 3',
|
|
31
|
+
'Topic :: Internet :: WWW/HTTP',
|
|
32
|
+
'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
|
|
33
|
+
],
|
|
34
|
+
install_requires=reqs,
|
|
35
|
+
)
|