dagster-hightouch 0.1.8__tar.gz → 0.28.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_hightouch-0.28.13/PKG-INFO +28 -0
- dagster_hightouch-0.28.13/dagster_hightouch/__init__.py +15 -0
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch/component.py +5 -15
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch/ops.py +5 -13
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch/resources.py +21 -51
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch/types.py +2 -2
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch/utils.py +1 -1
- dagster_hightouch-0.28.13/dagster_hightouch/version.py +1 -0
- dagster_hightouch-0.28.13/dagster_hightouch.egg-info/PKG-INFO +28 -0
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch.egg-info/SOURCES.txt +0 -1
- dagster_hightouch-0.28.13/dagster_hightouch.egg-info/entry_points.txt +2 -0
- dagster_hightouch-0.28.13/dagster_hightouch.egg-info/requires.txt +3 -0
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/setup.py +3 -4
- dagster_hightouch-0.1.8/PKG-INFO +0 -97
- dagster_hightouch-0.1.8/dagster_hightouch/__init__.py +0 -17
- dagster_hightouch-0.1.8/dagster_hightouch/version.py +0 -1
- dagster_hightouch-0.1.8/dagster_hightouch.egg-info/PKG-INFO +0 -97
- dagster_hightouch-0.1.8/dagster_hightouch.egg-info/entry_points.txt +0 -5
- dagster_hightouch-0.1.8/dagster_hightouch.egg-info/requires.txt +0 -3
- dagster_hightouch-0.1.8/pyproject.toml +0 -39
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/LICENSE +0 -0
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/MANIFEST.in +0 -0
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/README.md +0 -0
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch/py.typed +0 -0
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch.egg-info/dependency_links.txt +0 -0
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch.egg-info/not-zip-safe +0 -0
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch.egg-info/top_level.txt +0 -0
- {dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/setup.cfg +0 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dagster-hightouch
|
|
3
|
+
Version: 0.28.13
|
|
4
|
+
Summary: Package for integrating Hightouch with Dagster.
|
|
5
|
+
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-hightouch
|
|
6
|
+
Author: Dagster Labs
|
|
7
|
+
Author-email: hello@dagsterlabs.com
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Requires-Python: >=3.10,<3.14
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: dagster==1.12.13
|
|
18
|
+
Requires-Dist: requests
|
|
19
|
+
Requires-Dist: python-dateutil
|
|
20
|
+
Dynamic: author
|
|
21
|
+
Dynamic: author-email
|
|
22
|
+
Dynamic: classifier
|
|
23
|
+
Dynamic: home-page
|
|
24
|
+
Dynamic: license
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
Dynamic: requires-dist
|
|
27
|
+
Dynamic: requires-python
|
|
28
|
+
Dynamic: summary
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from dagster_shared.libraries import DagsterLibraryRegistry
|
|
2
|
+
|
|
3
|
+
from dagster_hightouch.component import HightouchSyncComponent
|
|
4
|
+
from dagster_hightouch.ops import hightouch_sync_op
|
|
5
|
+
from dagster_hightouch.resources import ConfigurableHightouchResource, HightouchResource
|
|
6
|
+
from dagster_hightouch.version import __version__ as __version__
|
|
7
|
+
|
|
8
|
+
DagsterLibraryRegistry.register("dagster-hightouch", __version__, is_dagster_package=True)
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"ConfigurableHightouchResource",
|
|
12
|
+
"HightouchResource",
|
|
13
|
+
"HightouchSyncComponent",
|
|
14
|
+
"hightouch_sync_op",
|
|
15
|
+
]
|
|
@@ -3,21 +3,14 @@ from collections.abc import Mapping
|
|
|
3
3
|
from typing import Any
|
|
4
4
|
|
|
5
5
|
import dagster as dg
|
|
6
|
-
from dagster.components import
|
|
7
|
-
Component,
|
|
8
|
-
Model,
|
|
9
|
-
Resolvable,
|
|
10
|
-
ResolvedAssetSpec,
|
|
11
|
-
ComponentLoadContext,
|
|
12
|
-
)
|
|
6
|
+
from dagster.components import Component, ComponentLoadContext, Model, Resolvable, ResolvedAssetSpec
|
|
13
7
|
from pydantic import Field
|
|
14
8
|
|
|
15
|
-
from .resources import ConfigurableHightouchResource
|
|
9
|
+
from dagster_hightouch.resources import ConfigurableHightouchResource
|
|
16
10
|
|
|
17
11
|
|
|
18
12
|
class HightouchSyncComponent(Component, Resolvable, Model):
|
|
19
|
-
"""
|
|
20
|
-
Represents a Hightouch sync as a Dagster asset.
|
|
13
|
+
"""Represents a Hightouch sync as a Dagster asset.
|
|
21
14
|
|
|
22
15
|
This component allows you to trigger a Hightouch sync and monitor its completion
|
|
23
16
|
as part of your Dagster asset graph. When materialized, it calls the Hightouch API,
|
|
@@ -69,8 +62,7 @@ class HightouchSyncComponent(Component, Resolvable, Model):
|
|
|
69
62
|
# Summing all failed rows (adds, changes, removes) for accurate reporting
|
|
70
63
|
failed_rows = result.sync_run_details.get("failedRows", {})
|
|
71
64
|
total_failed = sum(
|
|
72
|
-
failed_rows.get(k, 0)
|
|
73
|
-
for k in ["addedCount", "changedCount", "removedCount"]
|
|
65
|
+
failed_rows.get(k, 0) for k in ["addedCount", "changedCount", "removedCount"]
|
|
74
66
|
)
|
|
75
67
|
|
|
76
68
|
yield dg.MaterializeResult(
|
|
@@ -82,9 +74,7 @@ class HightouchSyncComponent(Component, Resolvable, Model):
|
|
|
82
74
|
"failed_adds": failed_rows.get("addedCount", 0),
|
|
83
75
|
"failed_changes": failed_rows.get("changedCount", 0),
|
|
84
76
|
"failed_removes": failed_rows.get("removedCount", 0),
|
|
85
|
-
"destination_details": dg.MetadataValue.json(
|
|
86
|
-
result.destination_details
|
|
87
|
-
),
|
|
77
|
+
"destination_details": dg.MetadataValue.json(result.destination_details),
|
|
88
78
|
"query_size": result.sync_run_details.get("querySize"),
|
|
89
79
|
"completion_ratio": result.sync_run_details.get("completionRatio"),
|
|
90
80
|
"failed_rows": result.sync_run_details.get("failedRows", {}).get(
|
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
from dagster import AssetMaterialization, Field, In, Noneable, Nothing, Out, Output, op
|
|
2
|
-
from dagster_hightouch.utils import (
|
|
3
|
-
generate_metadata_from_parsed_run,
|
|
4
|
-
parse_sync_run_details,
|
|
5
|
-
)
|
|
6
2
|
|
|
7
|
-
from .resources import DEFAULT_POLL_INTERVAL, HightouchOutput
|
|
3
|
+
from dagster_hightouch.resources import DEFAULT_POLL_INTERVAL, HightouchOutput
|
|
4
|
+
from dagster_hightouch.utils import generate_metadata_from_parsed_run, parse_sync_run_details
|
|
8
5
|
|
|
9
6
|
|
|
10
7
|
@op(
|
|
@@ -22,8 +19,7 @@ from .resources import DEFAULT_POLL_INTERVAL, HightouchOutput
|
|
|
22
19
|
"poll_interval": Field(
|
|
23
20
|
float,
|
|
24
21
|
default_value=DEFAULT_POLL_INTERVAL,
|
|
25
|
-
description="The time (in seconds) that will be waited between successive "
|
|
26
|
-
"polls.",
|
|
22
|
+
description="The time (in seconds) that will be waited between successive polls.",
|
|
27
23
|
),
|
|
28
24
|
"fail_on_warning": Field(
|
|
29
25
|
bool,
|
|
@@ -40,13 +36,11 @@ from .resources import DEFAULT_POLL_INTERVAL, HightouchOutput
|
|
|
40
36
|
tags={"kind": "hightouch"},
|
|
41
37
|
)
|
|
42
38
|
def hightouch_sync_op(context):
|
|
43
|
-
"""
|
|
44
|
-
Executes a Hightouch sync for a given ``sync_id``, and polls until that sync
|
|
39
|
+
"""Executes a Hightouch sync for a given ``sync_id``, and polls until that sync
|
|
45
40
|
completes, raising an error if it is unsuccessful. It outputs a HightouchOutput
|
|
46
41
|
which contains the details of the Hightouch connector after the sync run
|
|
47
42
|
successfully completes.
|
|
48
43
|
"""
|
|
49
|
-
|
|
50
44
|
hightouch_output: HightouchOutput = context.resources.hightouch.sync_and_poll(
|
|
51
45
|
sync_id=context.op_config["sync_id"],
|
|
52
46
|
fail_on_warning=context.op_config["fail_on_warning"],
|
|
@@ -55,9 +49,7 @@ def hightouch_sync_op(context):
|
|
|
55
49
|
)
|
|
56
50
|
destination_type = hightouch_output.destination_details.get("type")
|
|
57
51
|
destination_slug = hightouch_output.destination_details.get("slug")
|
|
58
|
-
sync_object = hightouch_output.sync_details.get("configuration", dict()).get(
|
|
59
|
-
"object"
|
|
60
|
-
)
|
|
52
|
+
sync_object = hightouch_output.sync_details.get("configuration", dict()).get("object")
|
|
61
53
|
if sync_object:
|
|
62
54
|
asset_name = ["hightouch", destination_type, destination_slug, sync_object]
|
|
63
55
|
else:
|
|
@@ -5,19 +5,11 @@ from typing import Any, Optional
|
|
|
5
5
|
from urllib.parse import urljoin
|
|
6
6
|
|
|
7
7
|
import requests
|
|
8
|
-
|
|
9
|
-
from dagster import (
|
|
10
|
-
Failure,
|
|
11
|
-
Field,
|
|
12
|
-
StringSource,
|
|
13
|
-
get_dagster_logger,
|
|
14
|
-
resource,
|
|
15
|
-
ConfigurableResource,
|
|
16
|
-
)
|
|
8
|
+
from dagster import ConfigurableResource, Failure, Field, StringSource, get_dagster_logger, resource
|
|
17
9
|
|
|
18
10
|
import dagster_hightouch
|
|
19
|
-
from
|
|
20
|
-
from .types import HightouchOutput
|
|
11
|
+
from dagster_hightouch import utils
|
|
12
|
+
from dagster_hightouch.types import HightouchOutput
|
|
21
13
|
|
|
22
14
|
HIGHTOUCH_API_BASE = "https://api.hightouch.io/api/v1/"
|
|
23
15
|
DEFAULT_POLL_INTERVAL = 3
|
|
@@ -34,8 +26,7 @@ WARNING = "warning"
|
|
|
34
26
|
|
|
35
27
|
|
|
36
28
|
class HightouchResource:
|
|
37
|
-
"""
|
|
38
|
-
Client for the Hightouch REST API.
|
|
29
|
+
"""Client for the Hightouch REST API.
|
|
39
30
|
|
|
40
31
|
This class provides methods to interface with Hightouch endpoints,
|
|
41
32
|
primarily for triggering syncs and polling for their status.
|
|
@@ -59,10 +50,8 @@ class HightouchResource:
|
|
|
59
50
|
def api_base_url(self) -> str:
|
|
60
51
|
return HIGHTOUCH_API_BASE
|
|
61
52
|
|
|
62
|
-
def make_request(
|
|
63
|
-
|
|
64
|
-
):
|
|
65
|
-
"""Creates and sends a request to the desired Hightouch API endpoint
|
|
53
|
+
def make_request(self, method: str, endpoint: str, params: dict[str, Any] | None = None):
|
|
54
|
+
"""Creates and sends a request to the desired Hightouch API endpoint.
|
|
66
55
|
|
|
67
56
|
Args:
|
|
68
57
|
method (str): The http method use for this request (e.g. "GET", "POST").
|
|
@@ -72,7 +61,6 @@ class HightouchResource:
|
|
|
72
61
|
Returns:
|
|
73
62
|
Dict[str, Any]: Parsed json data from the response to this request
|
|
74
63
|
"""
|
|
75
|
-
|
|
76
64
|
user_agent = f"HightouchDagsterOp/{dagster_hightouch.__version__}"
|
|
77
65
|
headers = {"Authorization": f"Bearer {self._api_key}", "User-Agent": user_agent}
|
|
78
66
|
|
|
@@ -97,9 +85,7 @@ class HightouchResource:
|
|
|
97
85
|
|
|
98
86
|
raise Failure("Exceeded max number of retries.")
|
|
99
87
|
|
|
100
|
-
def get_sync_run_details(
|
|
101
|
-
self, sync_id: str, sync_request_id: str
|
|
102
|
-
) -> list[dict[str, Any]]:
|
|
88
|
+
def get_sync_run_details(self, sync_id: str, sync_request_id: str) -> list[dict[str, Any]]:
|
|
103
89
|
"""Get details about a given sync run from the Hightouch API.
|
|
104
90
|
|
|
105
91
|
Args:
|
|
@@ -110,9 +96,7 @@ class HightouchResource:
|
|
|
110
96
|
Dict[str, Any]: Parsed json data from the response
|
|
111
97
|
"""
|
|
112
98
|
params = {"runId": sync_request_id}
|
|
113
|
-
return self.make_request(
|
|
114
|
-
method="GET", endpoint=f"syncs/{sync_id}/runs", params=params
|
|
115
|
-
)
|
|
99
|
+
return self.make_request(method="GET", endpoint=f"syncs/{sync_id}/runs", params=params)
|
|
116
100
|
|
|
117
101
|
def get_destination_details(self, destination_id: str) -> dict[str, Any]:
|
|
118
102
|
"""Get details about a destination from the Hightouch API.
|
|
@@ -123,9 +107,7 @@ class HightouchResource:
|
|
|
123
107
|
Returns:
|
|
124
108
|
Dict[str, Any]: Parsed json data from the response
|
|
125
109
|
"""
|
|
126
|
-
return self.make_request(
|
|
127
|
-
method="GET", endpoint=f"destinations/{destination_id}"
|
|
128
|
-
)
|
|
110
|
+
return self.make_request(method="GET", endpoint=f"destinations/{destination_id}")
|
|
129
111
|
|
|
130
112
|
def get_sync_details(self, sync_id: str) -> dict[str, Any]:
|
|
131
113
|
"""Get details about a given sync from the Hightouch API.
|
|
@@ -139,7 +121,7 @@ class HightouchResource:
|
|
|
139
121
|
return self.make_request(method="GET", endpoint=f"syncs/{sync_id}")
|
|
140
122
|
|
|
141
123
|
def start_sync(self, sync_id: str) -> str:
|
|
142
|
-
"""Trigger a sync and initiate a sync run
|
|
124
|
+
"""Trigger a sync and initiate a sync run.
|
|
143
125
|
|
|
144
126
|
Args:
|
|
145
127
|
sync_id (str): The Hightouch Sync ID.
|
|
@@ -147,9 +129,7 @@ class HightouchResource:
|
|
|
147
129
|
Returns:
|
|
148
130
|
str: The sync request ID created by the Hightouch API.
|
|
149
131
|
"""
|
|
150
|
-
return self.make_request(method="POST", endpoint=f"syncs/{sync_id}/trigger")[
|
|
151
|
-
"id"
|
|
152
|
-
]
|
|
132
|
+
return self.make_request(method="POST", endpoint=f"syncs/{sync_id}/trigger")["id"]
|
|
153
133
|
|
|
154
134
|
def poll_sync(
|
|
155
135
|
self,
|
|
@@ -159,7 +139,7 @@ class HightouchResource:
|
|
|
159
139
|
poll_interval: float = DEFAULT_POLL_INTERVAL,
|
|
160
140
|
poll_timeout: float | None = None,
|
|
161
141
|
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
|
162
|
-
"""Poll for the completion of a sync
|
|
142
|
+
"""Poll for the completion of a sync.
|
|
163
143
|
|
|
164
144
|
Args:
|
|
165
145
|
sync_id (str): The Hightouch Sync ID
|
|
@@ -204,10 +184,8 @@ class HightouchResource:
|
|
|
204
184
|
sync_id,
|
|
205
185
|
sync_request_id,
|
|
206
186
|
)
|
|
207
|
-
if (
|
|
208
|
-
poll_timeout
|
|
209
|
-
and datetime.datetime.now()
|
|
210
|
-
> poll_start + datetime.timedelta(seconds=poll_timeout)
|
|
187
|
+
if poll_timeout and datetime.datetime.now() > poll_start + datetime.timedelta(
|
|
188
|
+
seconds=poll_timeout
|
|
211
189
|
):
|
|
212
190
|
raise Failure(
|
|
213
191
|
f"Sync {sync_id} for request: {sync_request_id}' time out after "
|
|
@@ -216,9 +194,7 @@ class HightouchResource:
|
|
|
216
194
|
|
|
217
195
|
time.sleep(poll_interval)
|
|
218
196
|
sync_details = self.get_sync_details(sync_id)
|
|
219
|
-
destination_details = self.get_destination_details(
|
|
220
|
-
sync_details["destinationId"]
|
|
221
|
-
)
|
|
197
|
+
destination_details = self.get_destination_details(sync_details["destinationId"])
|
|
222
198
|
|
|
223
199
|
return (sync_details, sync_run_details, destination_details)
|
|
224
200
|
|
|
@@ -229,8 +205,7 @@ class HightouchResource:
|
|
|
229
205
|
poll_interval: float = DEFAULT_POLL_INTERVAL,
|
|
230
206
|
poll_timeout: float | None = None,
|
|
231
207
|
) -> HightouchOutput:
|
|
232
|
-
"""
|
|
233
|
-
Initialize a sync run for the given sync id, and polls until it completes
|
|
208
|
+
"""Initialize a sync run for the given sync id, and polls until it completes.
|
|
234
209
|
|
|
235
210
|
Args:
|
|
236
211
|
sync_id (str): The Hightouch Sync ID
|
|
@@ -239,6 +214,7 @@ class HightouchResource:
|
|
|
239
214
|
poll_interval (float): The time in seconds that will be waited between succcessive polls
|
|
240
215
|
poll_timeout (float): The maximum time that will be waited before this operation
|
|
241
216
|
times out.
|
|
217
|
+
|
|
242
218
|
Returns:
|
|
243
219
|
:py:class:`~HightouchOutput`:
|
|
244
220
|
Object containing details about the Hightouch sync run
|
|
@@ -277,8 +253,7 @@ class HightouchResource:
|
|
|
277
253
|
description="This resource helps manage Fivetran connectors",
|
|
278
254
|
)
|
|
279
255
|
def ht_resource(context) -> HightouchResource:
|
|
280
|
-
"""
|
|
281
|
-
This resource allows users to programatically interface with the Hightouch REST API to triggers
|
|
256
|
+
"""This resource allows users to programatically interface with the Hightouch REST API to triggers
|
|
282
257
|
syncs and monitor their progress.
|
|
283
258
|
"""
|
|
284
259
|
return HightouchResource(
|
|
@@ -290,8 +265,7 @@ def ht_resource(context) -> HightouchResource:
|
|
|
290
265
|
|
|
291
266
|
|
|
292
267
|
class ConfigurableHightouchResource(ConfigurableResource):
|
|
293
|
-
"""
|
|
294
|
-
A resource for connecting to the Hightouch API using pythonic configuration.
|
|
268
|
+
"""A resource for connecting to the Hightouch API using pythonic configuration.
|
|
295
269
|
|
|
296
270
|
This resource is the recommended way to manage Hightouch connections in
|
|
297
271
|
modern Dagster (1.0+).
|
|
@@ -326,10 +300,6 @@ class ConfigurableHightouchResource(ConfigurableResource):
|
|
|
326
300
|
fail_on_warning=fail_on_warning
|
|
327
301
|
if fail_on_warning is not None
|
|
328
302
|
else self.fail_on_warning,
|
|
329
|
-
poll_interval=poll_interval
|
|
330
|
-
if
|
|
331
|
-
else self.poll_interval,
|
|
332
|
-
poll_timeout=poll_timeout
|
|
333
|
-
if poll_timeout is not None
|
|
334
|
-
else self.poll_timeout,
|
|
303
|
+
poll_interval=poll_interval if poll_interval is not None else self.poll_interval,
|
|
304
|
+
poll_timeout=poll_timeout if poll_timeout is not None else self.poll_timeout,
|
|
335
305
|
)
|
|
@@ -12,8 +12,8 @@ class HightouchOutput(
|
|
|
12
12
|
],
|
|
13
13
|
)
|
|
14
14
|
):
|
|
15
|
-
"""
|
|
16
|
-
|
|
15
|
+
"""Contains recorded information about the state of a Hightouch sync after a sync completes.
|
|
16
|
+
|
|
17
17
|
Attributes:
|
|
18
18
|
sync_details (Dict[str, Any]):
|
|
19
19
|
https://hightouch.io/docs/api-reference/#operation/GetSync
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.28.13"
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dagster-hightouch
|
|
3
|
+
Version: 0.28.13
|
|
4
|
+
Summary: Package for integrating Hightouch with Dagster.
|
|
5
|
+
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-hightouch
|
|
6
|
+
Author: Dagster Labs
|
|
7
|
+
Author-email: hello@dagsterlabs.com
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Requires-Python: >=3.10,<3.14
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: dagster==1.12.13
|
|
18
|
+
Requires-Dist: requests
|
|
19
|
+
Requires-Dist: python-dateutil
|
|
20
|
+
Dynamic: author
|
|
21
|
+
Dynamic: author-email
|
|
22
|
+
Dynamic: classifier
|
|
23
|
+
Dynamic: home-page
|
|
24
|
+
Dynamic: license
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
Dynamic: requires-dist
|
|
27
|
+
Dynamic: requires-python
|
|
28
|
+
Dynamic: summary
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
|
+
|
|
2
3
|
from setuptools import find_packages, setup
|
|
3
4
|
|
|
4
5
|
|
|
5
6
|
def get_version() -> str:
|
|
6
7
|
version: dict[str, str] = {}
|
|
7
|
-
with open(
|
|
8
|
-
Path(__file__).parent / "dagster_hightouch/version.py", encoding="utf8"
|
|
9
|
-
) as fp:
|
|
8
|
+
with open(Path(__file__).parent / "dagster_hightouch/version.py", encoding="utf8") as fp:
|
|
10
9
|
exec(fp.read(), version)
|
|
11
10
|
return version["__version__"]
|
|
12
11
|
|
|
@@ -35,7 +34,7 @@ setup(
|
|
|
35
34
|
include_package_data=True,
|
|
36
35
|
python_requires=">=3.10,<3.14",
|
|
37
36
|
install_requires=[
|
|
38
|
-
"dagster==1.12.
|
|
37
|
+
"dagster==1.12.13",
|
|
39
38
|
"requests",
|
|
40
39
|
"python-dateutil",
|
|
41
40
|
],
|
dagster_hightouch-0.1.8/PKG-INFO
DELETED
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: dagster-hightouch
|
|
3
|
-
Version: 0.1.8
|
|
4
|
-
Summary: Dagster library for Hightouch
|
|
5
|
-
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-hightouch
|
|
6
|
-
Author: Dagster Labs
|
|
7
|
-
Author-email: hello@dagsterlabs.com
|
|
8
|
-
License: Apache-2.0
|
|
9
|
-
Requires-Python: >=3.10,<3.14
|
|
10
|
-
Description-Content-Type: text/markdown
|
|
11
|
-
License-File: LICENSE
|
|
12
|
-
Requires-Dist: dagster>=1.8.0
|
|
13
|
-
Requires-Dist: requests>=2.0
|
|
14
|
-
Requires-Dist: python-dateutil>=2.8.0
|
|
15
|
-
Dynamic: author
|
|
16
|
-
Dynamic: author-email
|
|
17
|
-
Dynamic: home-page
|
|
18
|
-
Dynamic: license-file
|
|
19
|
-
Dynamic: requires-python
|
|
20
|
-
|
|
21
|
-
## dagster-hightouch
|
|
22
|
-
|
|
23
|
-
A Dagster library for triggering syncs in Hightouch.
|
|
24
|
-
|
|
25
|
-
### Installation
|
|
26
|
-
|
|
27
|
-
To install the library, use pip alongside your existing Dagster environment.
|
|
28
|
-
|
|
29
|
-
```bash
|
|
30
|
-
pip install dagster-hightouch
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
### Configuration
|
|
34
|
-
|
|
35
|
-
First, you'll need to specify your [Hightouch API](https://hightouch.com/docs/developer-tools/api-guide/) key as a resource.
|
|
36
|
-
|
|
37
|
-
```python
|
|
38
|
-
# resources.py
|
|
39
|
-
from dagster_hightouch.resources import ht_resource as hightouch_resource
|
|
40
|
-
|
|
41
|
-
ht_resource = hightouch_resource.configured(
|
|
42
|
-
{"api_key": "555555-4444-3333-2222-1111111111"},
|
|
43
|
-
)
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
### Ops
|
|
47
|
-
|
|
48
|
-
The `hightouch_sync_op` will call the Hightouch API to trigger
|
|
49
|
-
a sync and monitor it until it completes.
|
|
50
|
-
|
|
51
|
-
```python
|
|
52
|
-
from dagster import ScheduleDefinition, get_dagster_logger, job
|
|
53
|
-
from dagster_hightouch.ops import hightouch_sync_op
|
|
54
|
-
from .resources import ht_resource
|
|
55
|
-
|
|
56
|
-
# Sync IDs are set as constants. You can also use
|
|
57
|
-
# the sync slug, read the documentation for other
|
|
58
|
-
# options.
|
|
59
|
-
|
|
60
|
-
HT_WS = "23620"
|
|
61
|
-
HT_ORG = "39619"
|
|
62
|
-
|
|
63
|
-
# We define two configured sync ops
|
|
64
|
-
run_ht_sync_workspaces = hightouch_sync_op.configured(
|
|
65
|
-
{"sync_id": HT_WS}, name="hightouch_sfdc_workspaces"
|
|
66
|
-
)
|
|
67
|
-
run_ht_sync_orgs = hightouch_sync_op.configured(
|
|
68
|
-
{"sync_id": HT_ORG}, name="hightouch_sfdc_organizations"
|
|
69
|
-
)
|
|
70
|
-
|
|
71
|
-
# And create a job with the defined resources, specifying the dependencies.
|
|
72
|
-
@job(
|
|
73
|
-
resource_defs={
|
|
74
|
-
"hightouch": ht_resource,
|
|
75
|
-
}
|
|
76
|
-
)
|
|
77
|
-
def ht_sfdc_job():
|
|
78
|
-
|
|
79
|
-
ht_orgs = run_ht_sync_orgs(start_after=ht_contacts)
|
|
80
|
-
run_ht_sync_workspaces(start_after=ht_orgs)
|
|
81
|
-
|
|
82
|
-
# And we schedule it to run every 30 mins.
|
|
83
|
-
every_30_schedule = ScheduleDefinition(job=ht_sfdc_job, cron_schedule="*/30 * * * *")
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
### Components (Modern)
|
|
87
|
-
|
|
88
|
-
You can now use Hightouch as a Dagster Component. This allows you to define syncs in YAML.
|
|
89
|
-
|
|
90
|
-
```yaml
|
|
91
|
-
# code/component.yaml
|
|
92
|
-
type: HightouchSyncComponent
|
|
93
|
-
attributes:
|
|
94
|
-
sync_id: "12345"
|
|
95
|
-
asset:
|
|
96
|
-
key: ["hightouch", "my_sync_asset"]
|
|
97
|
-
```
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
from dagster._core.libraries import DagsterLibraryRegistry
|
|
2
|
-
from .resources import ConfigurableHightouchResource, HightouchResource
|
|
3
|
-
from .ops import hightouch_sync_op
|
|
4
|
-
from .component import HightouchSyncComponent
|
|
5
|
-
|
|
6
|
-
__version__ = "0.1.8"
|
|
7
|
-
|
|
8
|
-
DagsterLibraryRegistry.register(
|
|
9
|
-
"dagster-hightouch", __version__, is_dagster_package=True
|
|
10
|
-
)
|
|
11
|
-
|
|
12
|
-
__all__ = [
|
|
13
|
-
"ConfigurableHightouchResource",
|
|
14
|
-
"HightouchResource",
|
|
15
|
-
"hightouch_sync_op",
|
|
16
|
-
"HightouchSyncComponent",
|
|
17
|
-
]
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.28.12"
|
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: dagster-hightouch
|
|
3
|
-
Version: 0.1.8
|
|
4
|
-
Summary: Dagster library for Hightouch
|
|
5
|
-
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-hightouch
|
|
6
|
-
Author: Dagster Labs
|
|
7
|
-
Author-email: hello@dagsterlabs.com
|
|
8
|
-
License: Apache-2.0
|
|
9
|
-
Requires-Python: >=3.10,<3.14
|
|
10
|
-
Description-Content-Type: text/markdown
|
|
11
|
-
License-File: LICENSE
|
|
12
|
-
Requires-Dist: dagster>=1.8.0
|
|
13
|
-
Requires-Dist: requests>=2.0
|
|
14
|
-
Requires-Dist: python-dateutil>=2.8.0
|
|
15
|
-
Dynamic: author
|
|
16
|
-
Dynamic: author-email
|
|
17
|
-
Dynamic: home-page
|
|
18
|
-
Dynamic: license-file
|
|
19
|
-
Dynamic: requires-python
|
|
20
|
-
|
|
21
|
-
## dagster-hightouch
|
|
22
|
-
|
|
23
|
-
A Dagster library for triggering syncs in Hightouch.
|
|
24
|
-
|
|
25
|
-
### Installation
|
|
26
|
-
|
|
27
|
-
To install the library, use pip alongside your existing Dagster environment.
|
|
28
|
-
|
|
29
|
-
```bash
|
|
30
|
-
pip install dagster-hightouch
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
### Configuration
|
|
34
|
-
|
|
35
|
-
First, you'll need to specify your [Hightouch API](https://hightouch.com/docs/developer-tools/api-guide/) key as a resource.
|
|
36
|
-
|
|
37
|
-
```python
|
|
38
|
-
# resources.py
|
|
39
|
-
from dagster_hightouch.resources import ht_resource as hightouch_resource
|
|
40
|
-
|
|
41
|
-
ht_resource = hightouch_resource.configured(
|
|
42
|
-
{"api_key": "555555-4444-3333-2222-1111111111"},
|
|
43
|
-
)
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
### Ops
|
|
47
|
-
|
|
48
|
-
The `hightouch_sync_op` will call the Hightouch API to trigger
|
|
49
|
-
a sync and monitor it until it completes.
|
|
50
|
-
|
|
51
|
-
```python
|
|
52
|
-
from dagster import ScheduleDefinition, get_dagster_logger, job
|
|
53
|
-
from dagster_hightouch.ops import hightouch_sync_op
|
|
54
|
-
from .resources import ht_resource
|
|
55
|
-
|
|
56
|
-
# Sync IDs are set as constants. You can also use
|
|
57
|
-
# the sync slug, read the documentation for other
|
|
58
|
-
# options.
|
|
59
|
-
|
|
60
|
-
HT_WS = "23620"
|
|
61
|
-
HT_ORG = "39619"
|
|
62
|
-
|
|
63
|
-
# We define two configured sync ops
|
|
64
|
-
run_ht_sync_workspaces = hightouch_sync_op.configured(
|
|
65
|
-
{"sync_id": HT_WS}, name="hightouch_sfdc_workspaces"
|
|
66
|
-
)
|
|
67
|
-
run_ht_sync_orgs = hightouch_sync_op.configured(
|
|
68
|
-
{"sync_id": HT_ORG}, name="hightouch_sfdc_organizations"
|
|
69
|
-
)
|
|
70
|
-
|
|
71
|
-
# And create a job with the defined resources, specifying the dependencies.
|
|
72
|
-
@job(
|
|
73
|
-
resource_defs={
|
|
74
|
-
"hightouch": ht_resource,
|
|
75
|
-
}
|
|
76
|
-
)
|
|
77
|
-
def ht_sfdc_job():
|
|
78
|
-
|
|
79
|
-
ht_orgs = run_ht_sync_orgs(start_after=ht_contacts)
|
|
80
|
-
run_ht_sync_workspaces(start_after=ht_orgs)
|
|
81
|
-
|
|
82
|
-
# And we schedule it to run every 30 mins.
|
|
83
|
-
every_30_schedule = ScheduleDefinition(job=ht_sfdc_job, cron_schedule="*/30 * * * *")
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
### Components (Modern)
|
|
87
|
-
|
|
88
|
-
You can now use Hightouch as a Dagster Component. This allows you to define syncs in YAML.
|
|
89
|
-
|
|
90
|
-
```yaml
|
|
91
|
-
# code/component.yaml
|
|
92
|
-
type: HightouchSyncComponent
|
|
93
|
-
attributes:
|
|
94
|
-
sync_id: "12345"
|
|
95
|
-
asset:
|
|
96
|
-
key: ["hightouch", "my_sync_asset"]
|
|
97
|
-
```
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
[project]
|
|
2
|
-
name = "dagster-hightouch"
|
|
3
|
-
description = "Dagster library for Hightouch"
|
|
4
|
-
readme = "README.md"
|
|
5
|
-
requires-python = ">=3.10"
|
|
6
|
-
license = { text = "Apache-2.0" }
|
|
7
|
-
dependencies = [
|
|
8
|
-
"dagster>=1.8.0",
|
|
9
|
-
"requests>=2.0",
|
|
10
|
-
"python-dateutil>=2.8.0",
|
|
11
|
-
]
|
|
12
|
-
dynamic = ["version"]
|
|
13
|
-
|
|
14
|
-
[project.entry-points."dagster.components"]
|
|
15
|
-
dagster_hightouch = "dagster_hightouch"
|
|
16
|
-
|
|
17
|
-
[project.entry-points."dagster_dg_cli.plugin"]
|
|
18
|
-
dagster_hightouch = "dagster_hightouch"
|
|
19
|
-
|
|
20
|
-
[tool.uv]
|
|
21
|
-
dev-dependencies = [
|
|
22
|
-
"pyright>=1.1.386",
|
|
23
|
-
"pytest",
|
|
24
|
-
"ruff==0.11.5",
|
|
25
|
-
"dagster-components",
|
|
26
|
-
]
|
|
27
|
-
|
|
28
|
-
[build-system]
|
|
29
|
-
requires = ["setuptools>=42"]
|
|
30
|
-
build-backend = "setuptools.build_meta"
|
|
31
|
-
|
|
32
|
-
[tool.setuptools]
|
|
33
|
-
packages = ["dagster_hightouch"]
|
|
34
|
-
|
|
35
|
-
[tool.setuptools.dynamic]
|
|
36
|
-
version = {attr = "dagster_hightouch.__version__"}
|
|
37
|
-
|
|
38
|
-
[tool.ruff]
|
|
39
|
-
required-version = "==0.11.5"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch.egg-info/not-zip-safe
RENAMED
|
File without changes
|
{dagster_hightouch-0.1.8 → dagster_hightouch-0.28.13}/dagster_hightouch.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|