dagster-hightouch 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_hightouch/__init__.py +4 -2
- dagster_hightouch/resources.py +8 -8
- dagster_hightouch/types.py +4 -4
- dagster_hightouch/utils.py +42 -28
- {dagster_hightouch-0.1.6.dist-info → dagster_hightouch-0.1.7.dist-info}/METADATA +2 -2
- dagster_hightouch-0.1.7.dist-info/RECORD +9 -0
- {dagster_hightouch-0.1.6.dist-info → dagster_hightouch-0.1.7.dist-info}/WHEEL +1 -1
- dagster_hightouch-0.1.6.dist-info/RECORD +0 -9
- {dagster_hightouch-0.1.6.dist-info → dagster_hightouch-0.1.7.dist-info}/top_level.txt +0 -0
dagster_hightouch/__init__.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
from dagster._core.libraries import DagsterLibraryRegistry
|
|
2
2
|
|
|
3
|
-
__version__ = "0.1.
|
|
3
|
+
__version__ = "0.1.7"
|
|
4
4
|
|
|
5
|
-
DagsterLibraryRegistry.register(
|
|
5
|
+
DagsterLibraryRegistry.register(
|
|
6
|
+
"dagster-hightouch", __version__, is_dagster_package=False
|
|
7
|
+
)
|
dagster_hightouch/resources.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import datetime
|
|
2
2
|
import logging
|
|
3
3
|
import time
|
|
4
|
-
from typing import Any
|
|
4
|
+
from typing import Any
|
|
5
5
|
from urllib.parse import urljoin
|
|
6
6
|
|
|
7
7
|
import requests
|
|
@@ -48,7 +48,7 @@ class HightouchResource:
|
|
|
48
48
|
return HIGHTOUCH_API_BASE
|
|
49
49
|
|
|
50
50
|
def make_request(
|
|
51
|
-
self, method: str, endpoint: str, params:
|
|
51
|
+
self, method: str, endpoint: str, params: dict[str, Any] | None = None
|
|
52
52
|
):
|
|
53
53
|
"""Creates and sends a request to the desired Hightouch API endpoint
|
|
54
54
|
|
|
@@ -87,7 +87,7 @@ class HightouchResource:
|
|
|
87
87
|
|
|
88
88
|
def get_sync_run_details(
|
|
89
89
|
self, sync_id: str, sync_request_id: str
|
|
90
|
-
) ->
|
|
90
|
+
) -> list[dict[str, Any]]:
|
|
91
91
|
"""Get details about a given sync run from the Hightouch API.
|
|
92
92
|
|
|
93
93
|
Args:
|
|
@@ -102,7 +102,7 @@ class HightouchResource:
|
|
|
102
102
|
method="GET", endpoint=f"syncs/{sync_id}/runs", params=params
|
|
103
103
|
)
|
|
104
104
|
|
|
105
|
-
def get_destination_details(self, destination_id: str) ->
|
|
105
|
+
def get_destination_details(self, destination_id: str) -> dict[str, Any]:
|
|
106
106
|
"""Get details about a destination from the Hightouch API.
|
|
107
107
|
|
|
108
108
|
Args:
|
|
@@ -115,7 +115,7 @@ class HightouchResource:
|
|
|
115
115
|
method="GET", endpoint=f"destinations/{destination_id}"
|
|
116
116
|
)
|
|
117
117
|
|
|
118
|
-
def get_sync_details(self, sync_id: str) ->
|
|
118
|
+
def get_sync_details(self, sync_id: str) -> dict[str, Any]:
|
|
119
119
|
"""Get details about a given sync from the Hightouch API.
|
|
120
120
|
|
|
121
121
|
Args:
|
|
@@ -145,8 +145,8 @@ class HightouchResource:
|
|
|
145
145
|
sync_request_id: str,
|
|
146
146
|
fail_on_warning: bool = False,
|
|
147
147
|
poll_interval: float = DEFAULT_POLL_INTERVAL,
|
|
148
|
-
poll_timeout:
|
|
149
|
-
) ->
|
|
148
|
+
poll_timeout: float | None = None,
|
|
149
|
+
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
|
150
150
|
"""Poll for the completion of a sync
|
|
151
151
|
|
|
152
152
|
Args:
|
|
@@ -215,7 +215,7 @@ class HightouchResource:
|
|
|
215
215
|
sync_id: str,
|
|
216
216
|
fail_on_warning: bool = False,
|
|
217
217
|
poll_interval: float = DEFAULT_POLL_INTERVAL,
|
|
218
|
-
poll_timeout:
|
|
218
|
+
poll_timeout: float | None = None,
|
|
219
219
|
) -> HightouchOutput:
|
|
220
220
|
"""
|
|
221
221
|
Initialize a sync run for the given sync id, and polls until it completes
|
dagster_hightouch/types.py
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
from collections import namedtuple
|
|
2
|
-
from typing import Any,
|
|
2
|
+
from typing import Any, NamedTuple
|
|
3
3
|
|
|
4
4
|
|
|
5
5
|
class HightouchOutput(
|
|
6
6
|
NamedTuple(
|
|
7
7
|
"_HightouchOutput",
|
|
8
8
|
[
|
|
9
|
-
("sync_details",
|
|
10
|
-
("sync_run_details",
|
|
11
|
-
("destination_details",
|
|
9
|
+
("sync_details", dict[str, Any]),
|
|
10
|
+
("sync_run_details", dict[str, Any]),
|
|
11
|
+
("destination_details", dict[str, Any]),
|
|
12
12
|
],
|
|
13
13
|
)
|
|
14
14
|
):
|
dagster_hightouch/utils.py
CHANGED
|
@@ -1,46 +1,60 @@
|
|
|
1
|
-
from typing import Type
|
|
2
|
-
|
|
3
1
|
from dateutil import parser
|
|
4
2
|
|
|
5
3
|
from .types import SyncRunParsedOutput
|
|
6
4
|
|
|
7
5
|
|
|
8
|
-
def parse_sync_run_details(sync_run_details) ->
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
x.finished_at = None
|
|
14
|
-
x.elapsed_seconds = None
|
|
6
|
+
def parse_sync_run_details(sync_run_details) -> SyncRunParsedOutput:
|
|
7
|
+
created_at = None
|
|
8
|
+
started_at = None
|
|
9
|
+
finished_at = None
|
|
10
|
+
elapsed_seconds = None
|
|
15
11
|
|
|
16
12
|
if sync_run_details.get("createdAt"):
|
|
17
|
-
|
|
13
|
+
created_at = parser.parse(sync_run_details["createdAt"])
|
|
18
14
|
if sync_run_details.get("startedAt"):
|
|
19
|
-
|
|
15
|
+
started_at = parser.parse(sync_run_details["startedAt"])
|
|
20
16
|
if sync_run_details.get("finishedAt"):
|
|
21
|
-
|
|
17
|
+
finished_at = parser.parse(sync_run_details["finishedAt"])
|
|
22
18
|
|
|
23
|
-
if
|
|
24
|
-
|
|
19
|
+
if finished_at and started_at:
|
|
20
|
+
elapsed_seconds = (finished_at - started_at).seconds
|
|
25
21
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
22
|
+
planned_add = sync_run_details["plannedRows"].get("addedCount")
|
|
23
|
+
planned_change = sync_run_details["plannedRows"].get("changedCount")
|
|
24
|
+
planned_remove = sync_run_details["plannedRows"].get("removedCount")
|
|
29
25
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
26
|
+
successful_add = sync_run_details["successfulRows"].get("addedCount")
|
|
27
|
+
successful_change = sync_run_details["successfulRows"].get("changedCount")
|
|
28
|
+
successful_remove = sync_run_details["successfulRows"].get("removedCount")
|
|
33
29
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
30
|
+
failed_add = sync_run_details["failedRows"].get("addedCount")
|
|
31
|
+
failed_change = sync_run_details["failedRows"].get("changedCount")
|
|
32
|
+
failed_remove = sync_run_details["failedRows"].get("removedCount")
|
|
37
33
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
34
|
+
query_size = sync_run_details.get("querySize")
|
|
35
|
+
status = sync_run_details.get("status")
|
|
36
|
+
completion_ratio = float(sync_run_details.get("completionRatio", 0))
|
|
37
|
+
error = sync_run_details.get("error")
|
|
42
38
|
|
|
43
|
-
return
|
|
39
|
+
return SyncRunParsedOutput(
|
|
40
|
+
created_at=created_at,
|
|
41
|
+
started_at=started_at,
|
|
42
|
+
finished_at=finished_at,
|
|
43
|
+
elapsed_seconds=elapsed_seconds,
|
|
44
|
+
planned_add=planned_add,
|
|
45
|
+
planned_change=planned_change,
|
|
46
|
+
planned_remove=planned_remove,
|
|
47
|
+
successful_add=successful_add,
|
|
48
|
+
successful_change=successful_change,
|
|
49
|
+
successful_remove=successful_remove,
|
|
50
|
+
failed_add=failed_add,
|
|
51
|
+
failed_change=failed_change,
|
|
52
|
+
failed_remove=failed_remove,
|
|
53
|
+
query_size=query_size,
|
|
54
|
+
status=status,
|
|
55
|
+
completion_ratio=completion_ratio,
|
|
56
|
+
error=error,
|
|
57
|
+
)
|
|
44
58
|
|
|
45
59
|
|
|
46
60
|
def generate_metadata_from_parsed_run(parsed_output: SyncRunParsedOutput):
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-hightouch
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.7
|
|
4
4
|
Summary: Dagster library for Hightouch
|
|
5
|
-
Requires-Python: >=3.
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: dagster>=0.14
|
|
8
8
|
Requires-Dist: requests>=2.0
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
dagster_hightouch/__init__.py,sha256=j6plWaGtVFecF4O7wHji63WP13pQzWHvtmTmqPzDtdw,181
|
|
2
|
+
dagster_hightouch/ops.py,sha256=8_NMNKlj1VzFLoiBA__tn7y9qghtijhuLuKdtKkA4Ak,2951
|
|
3
|
+
dagster_hightouch/resources.py,sha256=HodHBIzIlOKF51EYCvomkiXHFfyjFFYdtwZE2Vw1VKw,9887
|
|
4
|
+
dagster_hightouch/types.py,sha256=nI7BTLpXCvYM3HUQLK3nxa1FKnl-G0_bJIHp2BYialc,1286
|
|
5
|
+
dagster_hightouch/utils.py,sha256=-mSN0aglUIwsxBiBRgj0cHi5B5SVXWk3K-rezMyPKaM,2866
|
|
6
|
+
dagster_hightouch-0.1.7.dist-info/METADATA,sha256=qQvA_h8BflOIGtLXk3fZVkVgGNqKJCxtufuvecC58mA,1876
|
|
7
|
+
dagster_hightouch-0.1.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
8
|
+
dagster_hightouch-0.1.7.dist-info/top_level.txt,sha256=oMoqAW8f7myyEz6FG5zNzVQr3RzPdYm67ekvSD96N54,18
|
|
9
|
+
dagster_hightouch-0.1.7.dist-info/RECORD,,
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
dagster_hightouch/__init__.py,sha256=tFOTsn7KT2BwcOOTaWIRP77LdPthYbF8_AC2Auirfys,149
|
|
2
|
-
dagster_hightouch/ops.py,sha256=8_NMNKlj1VzFLoiBA__tn7y9qghtijhuLuKdtKkA4Ak,2951
|
|
3
|
-
dagster_hightouch/resources.py,sha256=7ATTOkSGeUN8vSzcyjnB0cFfndXisYmE2dI4h4xKPbs,9925
|
|
4
|
-
dagster_hightouch/types.py,sha256=d8LOgAKL4JxfYw5HIHPjk5_yvlYrEFKyyBIz4YqzPLE,1292
|
|
5
|
-
dagster_hightouch/utils.py,sha256=ib_t7hpzsgqEUnEku95rdH2QtyqVEGoCMeBxnfPGxHg,2352
|
|
6
|
-
dagster_hightouch-0.1.6.dist-info/METADATA,sha256=vFTx5sNvpsw_pc4-cV5mGhvetbKhaWtYTwCm6X_KBkk,1875
|
|
7
|
-
dagster_hightouch-0.1.6.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
|
8
|
-
dagster_hightouch-0.1.6.dist-info/top_level.txt,sha256=oMoqAW8f7myyEz6FG5zNzVQr3RzPdYm67ekvSD96N54,18
|
|
9
|
-
dagster_hightouch-0.1.6.dist-info/RECORD,,
|
|
File without changes
|