fast2dev-dbt-runner 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fast2dev_dbt_runner/__init__.py +47 -0
- fast2dev_dbt_runner/_compat.py +41 -0
- fast2dev_dbt_runner/airbyte_task_group_builder.py +259 -0
- fast2dev_dbt_runner/bash_operator/__init__.py +3 -0
- fast2dev_dbt_runner/bash_operator/datawarehouse_secrets.py +490 -0
- fast2dev_dbt_runner/bash_operator/dbt_hook.py +615 -0
- fast2dev_dbt_runner/bash_operator/dbt_operator.py +248 -0
- fast2dev_dbt_runner/cached_manifest_loader.py +275 -0
- fast2dev_dbt_runner/dbt_manifest_parser_api_operator.py +452 -0
- fast2dev_dbt_runner/dbt_manifest_parser_bash_operator.py +361 -0
- fast2dev_dbt_runner/dbt_manifest_parser_gke_operator.py +401 -0
- fast2dev_dbt_runner/dbt_manifest_parser_k8s_operator.py +448 -0
- fast2dev_dbt_runner/utils.py +451 -0
- fast2dev_dbt_runner-0.0.1.dist-info/METADATA +398 -0
- fast2dev_dbt_runner-0.0.1.dist-info/RECORD +18 -0
- fast2dev_dbt_runner-0.0.1.dist-info/WHEEL +5 -0
- fast2dev_dbt_runner-0.0.1.dist-info/licenses/LICENSE +21 -0
- fast2dev_dbt_runner-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Fast2.Dev dbt runner — Airflow operators generated from a dbt manifest."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("fast2dev-dbt-runner")
|
|
7
|
+
except PackageNotFoundError:
|
|
8
|
+
__version__ = "0.0.1"
|
|
9
|
+
|
|
10
|
+
from fast2dev_dbt_runner.cached_manifest_loader import (
|
|
11
|
+
clear_cache,
|
|
12
|
+
get_cache_stats,
|
|
13
|
+
load_dbt_manifest_cached,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"__version__",
|
|
18
|
+
"clear_cache",
|
|
19
|
+
"get_cache_stats",
|
|
20
|
+
"load_dbt_manifest_cached",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
from fast2dev_dbt_runner.airbyte_task_group_builder import TaskBuilder
|
|
25
|
+
from fast2dev_dbt_runner.dbt_manifest_parser_api_operator import (
|
|
26
|
+
DbtManifestParser as DbtManifestParserApiOperator,
|
|
27
|
+
)
|
|
28
|
+
from fast2dev_dbt_runner.dbt_manifest_parser_bash_operator import (
|
|
29
|
+
DbtManifestParser as DbtManifestParserBashOperator,
|
|
30
|
+
)
|
|
31
|
+
from fast2dev_dbt_runner.dbt_manifest_parser_gke_operator import (
|
|
32
|
+
DbtManifestParser as DbtManifestParserGkeOperator,
|
|
33
|
+
)
|
|
34
|
+
from fast2dev_dbt_runner.dbt_manifest_parser_k8s_operator import (
|
|
35
|
+
DbtManifestParser as DbtManifestParserK8sOperator,
|
|
36
|
+
)
|
|
37
|
+
except ImportError:
|
|
38
|
+
# Airflow is optional at import time; operators load when the extra is installed.
|
|
39
|
+
pass
|
|
40
|
+
else:
|
|
41
|
+
__all__ += [
|
|
42
|
+
"DbtManifestParserApiOperator",
|
|
43
|
+
"DbtManifestParserBashOperator",
|
|
44
|
+
"DbtManifestParserGkeOperator",
|
|
45
|
+
"DbtManifestParserK8sOperator",
|
|
46
|
+
"TaskBuilder",
|
|
47
|
+
]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Airflow 3.2 import shims. Cluster is v3-only; fallbacks keep unit tests importable."""
|
|
2
|
+
|
|
3
|
+
try:
|
|
4
|
+
from airflow.sdk import BaseOperator, TaskGroup
|
|
5
|
+
except ImportError:
|
|
6
|
+
from airflow.models import BaseOperator
|
|
7
|
+
from airflow.utils.task_group import TaskGroup
|
|
8
|
+
|
|
9
|
+
try:
|
|
10
|
+
from airflow.sdk.bases.hook import BaseHook
|
|
11
|
+
except ImportError:
|
|
12
|
+
from airflow.hooks.base import BaseHook
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
from airflow.sdk.exceptions import AirflowException, AirflowFailException
|
|
16
|
+
except ImportError:
|
|
17
|
+
from airflow.exceptions import AirflowException, AirflowFailException
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
from airflow.providers.standard.operators.empty import EmptyOperator
|
|
21
|
+
except ImportError:
|
|
22
|
+
from airflow.operators.empty import EmptyOperator
|
|
23
|
+
|
|
24
|
+
try:
|
|
25
|
+
from airflow.providers.standard.operators.python import (
|
|
26
|
+
BranchPythonOperator,
|
|
27
|
+
PythonOperator,
|
|
28
|
+
)
|
|
29
|
+
except ImportError:
|
|
30
|
+
from airflow.operators.python import BranchPythonOperator, PythonOperator
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"AirflowException",
|
|
34
|
+
"AirflowFailException",
|
|
35
|
+
"BaseHook",
|
|
36
|
+
"BaseOperator",
|
|
37
|
+
"BranchPythonOperator",
|
|
38
|
+
"EmptyOperator",
|
|
39
|
+
"PythonOperator",
|
|
40
|
+
"TaskGroup",
|
|
41
|
+
]
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from datetime import datetime, timedelta
|
|
3
|
+
|
|
4
|
+
import pytz
|
|
5
|
+
import requests
|
|
6
|
+
from fast2dev_dbt_runner._compat import (
|
|
7
|
+
BaseHook,
|
|
8
|
+
BranchPythonOperator,
|
|
9
|
+
EmptyOperator,
|
|
10
|
+
PythonOperator,
|
|
11
|
+
TaskGroup,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_con_info(conn, api, ti=None):
|
|
16
|
+
url = f"{api}/api/v1/connections/get"
|
|
17
|
+
headers = {"accept": "application/json"}
|
|
18
|
+
data = {"connectionId": conn}
|
|
19
|
+
response = requests.post(url, headers=headers, json=data)
|
|
20
|
+
|
|
21
|
+
if response.status_code == 200:
|
|
22
|
+
response_json = response.json()
|
|
23
|
+
name = response_json.get("name")
|
|
24
|
+
print(f"Connection name is: {name}")
|
|
25
|
+
|
|
26
|
+
schedule_type = response_json.get("scheduleType")
|
|
27
|
+
ti.xcom_push(key="schedule_type", value=schedule_type)
|
|
28
|
+
print("Schedule type is: ", schedule_type)
|
|
29
|
+
return schedule_type
|
|
30
|
+
else:
|
|
31
|
+
print(f"Error: {response.status_code} - {response.text}")
|
|
32
|
+
raise Exception(f"Error: {response.status_code} - {response.text}")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_con_status(conn, api, ti=None):
|
|
36
|
+
url = f"{api}/api/v1/connections/status"
|
|
37
|
+
headers = {"accept": "application/json"}
|
|
38
|
+
data = {"connectionIds": [conn]}
|
|
39
|
+
response = requests.post(url, headers=headers, json=data)
|
|
40
|
+
attempt = 1
|
|
41
|
+
|
|
42
|
+
while True:
|
|
43
|
+
if response.status_code == 200:
|
|
44
|
+
response_json = response.json()[0]
|
|
45
|
+
|
|
46
|
+
# Check both old and new API response formats
|
|
47
|
+
is_running = response_json.get("isRunning")
|
|
48
|
+
conn_status = response_json.get("connectionSyncStatus")
|
|
49
|
+
|
|
50
|
+
# For new API format
|
|
51
|
+
if conn_status is not None:
|
|
52
|
+
is_running = conn_status == "running"
|
|
53
|
+
|
|
54
|
+
print(f"Check: {attempt}, Currently running: {is_running}")
|
|
55
|
+
|
|
56
|
+
if is_running is False or conn_status == "synced":
|
|
57
|
+
last_successful_sync_sec = response_json.get("lastSuccessfulSync")
|
|
58
|
+
if ti:
|
|
59
|
+
ti.xcom_push(
|
|
60
|
+
key="last_successful_sync_sec", value=last_successful_sync_sec
|
|
61
|
+
)
|
|
62
|
+
print("Last successful sync in sec: ", last_successful_sync_sec)
|
|
63
|
+
return True
|
|
64
|
+
else:
|
|
65
|
+
print(f"Error: {response.status_code} - {response.text}")
|
|
66
|
+
raise Exception(f"Error: {response.status_code} - {response.text}")
|
|
67
|
+
|
|
68
|
+
time.sleep(30)
|
|
69
|
+
attempt += 1
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def trigger_sync_job(conn, api, ti=None):
|
|
73
|
+
# Old API endpoint
|
|
74
|
+
url = f"{api}/api/v1/connections/sync"
|
|
75
|
+
headers = {"accept": "application/json"}
|
|
76
|
+
data = {"connectionId": conn}
|
|
77
|
+
|
|
78
|
+
try:
|
|
79
|
+
response = requests.post(url, headers=headers, json=data)
|
|
80
|
+
|
|
81
|
+
# Handle case where sync is already running (409 Conflict)
|
|
82
|
+
if response.status_code == 409:
|
|
83
|
+
print(f"Sync already running for connection {conn}. Waiting for it to complete...")
|
|
84
|
+
# Wait for the existing sync to complete
|
|
85
|
+
get_con_status(conn, api, ti)
|
|
86
|
+
# Get the latest job ID for this connection
|
|
87
|
+
url = f"{api}/api/v1/jobs/list"
|
|
88
|
+
list_data = {"connectionId": conn, "limit": 1}
|
|
89
|
+
list_response = requests.post(url, headers=headers, json=list_data)
|
|
90
|
+
|
|
91
|
+
if list_response.status_code == 200:
|
|
92
|
+
jobs = list_response.json().get("jobs", [])
|
|
93
|
+
if jobs:
|
|
94
|
+
job_id = jobs[0].get("id")
|
|
95
|
+
ti.xcom_push(key="job_id", value=job_id)
|
|
96
|
+
print("Retrieved existing job with id: ", job_id)
|
|
97
|
+
return job_id
|
|
98
|
+
raise Exception("Failed to retrieve existing job information")
|
|
99
|
+
|
|
100
|
+
elif response.status_code == 200:
|
|
101
|
+
response_json = response.json()
|
|
102
|
+
job = response_json.get("job")
|
|
103
|
+
job_id = job.get("id")
|
|
104
|
+
ti.xcom_push(key="job_id", value=job_id)
|
|
105
|
+
print("Triggered new job with id: ", job_id)
|
|
106
|
+
return job_id
|
|
107
|
+
else:
|
|
108
|
+
print(f"Error: {response.status_code} - {response.text}")
|
|
109
|
+
raise Exception(f"Error: {response.status_code} - {response.text}")
|
|
110
|
+
|
|
111
|
+
except requests.exceptions.RequestException as e:
|
|
112
|
+
print(f"Request failed: {str(e)}")
|
|
113
|
+
raise Exception(f"Failed to trigger sync: {str(e)}")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def skip_if_already_run(i, ti=None, data_interval_end=None):
|
|
117
|
+
last_successful_sync_sec = ti.xcom_pull(
|
|
118
|
+
task_ids=f"airbyte.check_connections_status_{i}", key="last_successful_sync_sec"
|
|
119
|
+
)
|
|
120
|
+
d = datetime.utcfromtimestamp(int(last_successful_sync_sec))
|
|
121
|
+
last_sync = datetime(
|
|
122
|
+
year=d.year,
|
|
123
|
+
month=d.month,
|
|
124
|
+
day=d.day,
|
|
125
|
+
hour=d.hour,
|
|
126
|
+
minute=d.minute,
|
|
127
|
+
second=d.second,
|
|
128
|
+
tzinfo=pytz.UTC,
|
|
129
|
+
)
|
|
130
|
+
print("end of running date: ", data_interval_end)
|
|
131
|
+
print("last connection sync: ", last_sync)
|
|
132
|
+
diff_to_prev_run_in_hours = (data_interval_end - last_sync).total_seconds() / 3600
|
|
133
|
+
hours = timedelta(hours=diff_to_prev_run_in_hours)
|
|
134
|
+
print("Previous connection sync was: ", hours, " hours ago")
|
|
135
|
+
skip_next_task = hours <= timedelta(hours=1)
|
|
136
|
+
if skip_next_task:
|
|
137
|
+
return "airbyte.finish_airbyte"
|
|
138
|
+
else:
|
|
139
|
+
return f"airbyte.trigger_sync_{i}"
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def skip_if_not_manual(i, ti=None):
|
|
143
|
+
schedule_type = ti.xcom_pull(
|
|
144
|
+
task_ids=f"airbyte.get_connections_info_{i}", key="schedule_type"
|
|
145
|
+
)
|
|
146
|
+
skip_next_task = schedule_type != "manual"
|
|
147
|
+
if skip_next_task:
|
|
148
|
+
print(f"Schedule type should be 'manual' to proceed, but it {schedule_type}!")
|
|
149
|
+
return "airbyte.finish_airbyte"
|
|
150
|
+
else:
|
|
151
|
+
return f"airbyte.check_connections_status_{i}"
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def check_job(i, api, ti=None):
|
|
155
|
+
job_id = ti.xcom_pull(task_ids=f"airbyte.trigger_sync_{i}", key="job_id")
|
|
156
|
+
attempt = 1
|
|
157
|
+
while True:
|
|
158
|
+
url = f"{api}/api/v1/jobs/get"
|
|
159
|
+
headers = {"accept": "application/json"}
|
|
160
|
+
data = {"id": job_id}
|
|
161
|
+
response = requests.post(url, headers=headers, json=data)
|
|
162
|
+
if response.status_code == 200:
|
|
163
|
+
response_json = response.json()
|
|
164
|
+
job = response_json.get("job")
|
|
165
|
+
status = job.get("status")
|
|
166
|
+
print(f"Check: {attempt}, status: {status}")
|
|
167
|
+
if status in ("failed", 'cancelled', 'incomplete'):
|
|
168
|
+
raise Exception(f"Airbyte Job {job_id}: {status}.")
|
|
169
|
+
elif status == 'succeeded':
|
|
170
|
+
return True
|
|
171
|
+
|
|
172
|
+
else:
|
|
173
|
+
print(f"Error: {response.status_code} - {response.text}")
|
|
174
|
+
raise Exception(f"Error: {response.status_code} - {response.text}")
|
|
175
|
+
|
|
176
|
+
# Wait for 30 sec before making the next API call
|
|
177
|
+
time.sleep(30)
|
|
178
|
+
attempt += 1
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class TaskBuilder:
|
|
182
|
+
def __init__(self, connection_ids) -> None:
|
|
183
|
+
self.airbyte_connection_ids = connection_ids
|
|
184
|
+
|
|
185
|
+
def build_tasks(self, connection_ids):
|
|
186
|
+
|
|
187
|
+
airbyte_conn = BaseHook.get_connection('data_replication_conn')
|
|
188
|
+
|
|
189
|
+
# Default host
|
|
190
|
+
default_host = 'data-replication-airbyte-webapp-svc.data-replication.svc.cluster.local'
|
|
191
|
+
|
|
192
|
+
host = default_host # Start with the default host
|
|
193
|
+
|
|
194
|
+
if airbyte_conn:
|
|
195
|
+
if airbyte_conn.host:
|
|
196
|
+
host = airbyte_conn.host
|
|
197
|
+
elif airbyte_conn.extra:
|
|
198
|
+
extra = airbyte_conn.extra_dejson
|
|
199
|
+
if 'host' in extra and extra['host']:
|
|
200
|
+
host = extra['host']
|
|
201
|
+
|
|
202
|
+
with TaskGroup(group_id="airbyte") as airbyte_group:
|
|
203
|
+
start_airbyte = EmptyOperator(task_id="start_airbyte")
|
|
204
|
+
finish_airbyte = EmptyOperator(
|
|
205
|
+
task_id="finish_airbyte", trigger_rule="none_failed_min_one_success"
|
|
206
|
+
)
|
|
207
|
+
for _, connection in enumerate(connection_ids):
|
|
208
|
+
index = connection.rsplit("-", 1)[-1]
|
|
209
|
+
|
|
210
|
+
task_get_connections_info = PythonOperator(
|
|
211
|
+
task_id=f"get_connections_info_{index}",
|
|
212
|
+
python_callable=get_con_info,
|
|
213
|
+
op_kwargs={"conn": connection, "api": host},
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
task_check_connection_schedule_type = BranchPythonOperator(
|
|
217
|
+
task_id=f"check_connection_schedule_type_{index}",
|
|
218
|
+
python_callable=skip_if_not_manual,
|
|
219
|
+
op_kwargs={"i": index},
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
task_check_connections_status = PythonOperator(
|
|
223
|
+
task_id=f"check_connections_status_{index}",
|
|
224
|
+
python_callable=get_con_status,
|
|
225
|
+
op_kwargs={"conn": connection, "api": host},
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
task_check_connections_last_sync = BranchPythonOperator(
|
|
229
|
+
task_id=f"check_connections_last_sync_{index}",
|
|
230
|
+
python_callable=skip_if_already_run,
|
|
231
|
+
op_kwargs={"i": index},
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
task_trigger_sync = PythonOperator(
|
|
235
|
+
task_id=f"trigger_sync_{index}",
|
|
236
|
+
python_callable=trigger_sync_job,
|
|
237
|
+
op_kwargs={"conn": connection, "api": host},
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
task_check_job = PythonOperator(
|
|
241
|
+
task_id=f"check_job_{index}",
|
|
242
|
+
python_callable=check_job,
|
|
243
|
+
op_kwargs={"i": index, "api": host},
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
(
|
|
247
|
+
start_airbyte
|
|
248
|
+
>> task_get_connections_info
|
|
249
|
+
>> task_check_connection_schedule_type
|
|
250
|
+
>> [finish_airbyte, task_check_connections_status]
|
|
251
|
+
)
|
|
252
|
+
(
|
|
253
|
+
task_check_connections_status
|
|
254
|
+
>> task_check_connections_last_sync
|
|
255
|
+
>> [finish_airbyte, task_trigger_sync]
|
|
256
|
+
)
|
|
257
|
+
task_trigger_sync >> task_check_job >> finish_airbyte
|
|
258
|
+
|
|
259
|
+
return airbyte_group
|