fast2dev-dbt-runner 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,47 @@
1
+ """Fast2.Dev dbt runner — Airflow operators generated from a dbt manifest."""
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ try:
6
+ __version__ = version("fast2dev-dbt-runner")
7
+ except PackageNotFoundError:
8
+ __version__ = "0.0.1"
9
+
10
+ from fast2dev_dbt_runner.cached_manifest_loader import (
11
+ clear_cache,
12
+ get_cache_stats,
13
+ load_dbt_manifest_cached,
14
+ )
15
+
16
+ __all__ = [
17
+ "__version__",
18
+ "clear_cache",
19
+ "get_cache_stats",
20
+ "load_dbt_manifest_cached",
21
+ ]
22
+
23
+ try:
24
+ from fast2dev_dbt_runner.airbyte_task_group_builder import TaskBuilder
25
+ from fast2dev_dbt_runner.dbt_manifest_parser_api_operator import (
26
+ DbtManifestParser as DbtManifestParserApiOperator,
27
+ )
28
+ from fast2dev_dbt_runner.dbt_manifest_parser_bash_operator import (
29
+ DbtManifestParser as DbtManifestParserBashOperator,
30
+ )
31
+ from fast2dev_dbt_runner.dbt_manifest_parser_gke_operator import (
32
+ DbtManifestParser as DbtManifestParserGkeOperator,
33
+ )
34
+ from fast2dev_dbt_runner.dbt_manifest_parser_k8s_operator import (
35
+ DbtManifestParser as DbtManifestParserK8sOperator,
36
+ )
37
+ except ImportError:
38
+ # Airflow is optional at import time; operators load when the extra is installed.
39
+ pass
40
+ else:
41
+ __all__ += [
42
+ "DbtManifestParserApiOperator",
43
+ "DbtManifestParserBashOperator",
44
+ "DbtManifestParserGkeOperator",
45
+ "DbtManifestParserK8sOperator",
46
+ "TaskBuilder",
47
+ ]
@@ -0,0 +1,41 @@
1
+ """Airflow 3.2 import shims. Cluster is v3-only; fallbacks keep unit tests importable."""
2
+
3
+ try:
4
+ from airflow.sdk import BaseOperator, TaskGroup
5
+ except ImportError:
6
+ from airflow.models import BaseOperator
7
+ from airflow.utils.task_group import TaskGroup
8
+
9
+ try:
10
+ from airflow.sdk.bases.hook import BaseHook
11
+ except ImportError:
12
+ from airflow.hooks.base import BaseHook
13
+
14
+ try:
15
+ from airflow.sdk.exceptions import AirflowException, AirflowFailException
16
+ except ImportError:
17
+ from airflow.exceptions import AirflowException, AirflowFailException
18
+
19
+ try:
20
+ from airflow.providers.standard.operators.empty import EmptyOperator
21
+ except ImportError:
22
+ from airflow.operators.empty import EmptyOperator
23
+
24
+ try:
25
+ from airflow.providers.standard.operators.python import (
26
+ BranchPythonOperator,
27
+ PythonOperator,
28
+ )
29
+ except ImportError:
30
+ from airflow.operators.python import BranchPythonOperator, PythonOperator
31
+
32
+ __all__ = [
33
+ "AirflowException",
34
+ "AirflowFailException",
35
+ "BaseHook",
36
+ "BaseOperator",
37
+ "BranchPythonOperator",
38
+ "EmptyOperator",
39
+ "PythonOperator",
40
+ "TaskGroup",
41
+ ]
@@ -0,0 +1,259 @@
1
+ import time
2
+ from datetime import datetime, timedelta
3
+
4
+ import pytz
5
+ import requests
6
+ from fast2dev_dbt_runner._compat import (
7
+ BaseHook,
8
+ BranchPythonOperator,
9
+ EmptyOperator,
10
+ PythonOperator,
11
+ TaskGroup,
12
+ )
13
+
14
+
15
+ def get_con_info(conn, api, ti=None):
16
+ url = f"{api}/api/v1/connections/get"
17
+ headers = {"accept": "application/json"}
18
+ data = {"connectionId": conn}
19
+ response = requests.post(url, headers=headers, json=data)
20
+
21
+ if response.status_code == 200:
22
+ response_json = response.json()
23
+ name = response_json.get("name")
24
+ print(f"Connection name is: {name}")
25
+
26
+ schedule_type = response_json.get("scheduleType")
27
+ ti.xcom_push(key="schedule_type", value=schedule_type)
28
+ print("Schedule type is: ", schedule_type)
29
+ return schedule_type
30
+ else:
31
+ print(f"Error: {response.status_code} - {response.text}")
32
+ raise Exception(f"Error: {response.status_code} - {response.text}")
33
+
34
+
35
+ def get_con_status(conn, api, ti=None):
36
+ url = f"{api}/api/v1/connections/status"
37
+ headers = {"accept": "application/json"}
38
+ data = {"connectionIds": [conn]}
39
+ response = requests.post(url, headers=headers, json=data)
40
+ attempt = 1
41
+
42
+ while True:
43
+ if response.status_code == 200:
44
+ response_json = response.json()[0]
45
+
46
+ # Check both old and new API response formats
47
+ is_running = response_json.get("isRunning")
48
+ conn_status = response_json.get("connectionSyncStatus")
49
+
50
+ # For new API format
51
+ if conn_status is not None:
52
+ is_running = conn_status == "running"
53
+
54
+ print(f"Check: {attempt}, Currently running: {is_running}")
55
+
56
+ if is_running is False or conn_status == "synced":
57
+ last_successful_sync_sec = response_json.get("lastSuccessfulSync")
58
+ if ti:
59
+ ti.xcom_push(
60
+ key="last_successful_sync_sec", value=last_successful_sync_sec
61
+ )
62
+ print("Last successful sync in sec: ", last_successful_sync_sec)
63
+ return True
64
+ else:
65
+ print(f"Error: {response.status_code} - {response.text}")
66
+ raise Exception(f"Error: {response.status_code} - {response.text}")
67
+
68
+ time.sleep(30)
69
+ attempt += 1
70
+
71
+
72
+ def trigger_sync_job(conn, api, ti=None):
73
+ # Old API endpoint
74
+ url = f"{api}/api/v1/connections/sync"
75
+ headers = {"accept": "application/json"}
76
+ data = {"connectionId": conn}
77
+
78
+ try:
79
+ response = requests.post(url, headers=headers, json=data)
80
+
81
+ # Handle case where sync is already running (409 Conflict)
82
+ if response.status_code == 409:
83
+ print(f"Sync already running for connection {conn}. Waiting for it to complete...")
84
+ # Wait for the existing sync to complete
85
+ get_con_status(conn, api, ti)
86
+ # Get the latest job ID for this connection
87
+ url = f"{api}/api/v1/jobs/list"
88
+ list_data = {"connectionId": conn, "limit": 1}
89
+ list_response = requests.post(url, headers=headers, json=list_data)
90
+
91
+ if list_response.status_code == 200:
92
+ jobs = list_response.json().get("jobs", [])
93
+ if jobs:
94
+ job_id = jobs[0].get("id")
95
+ ti.xcom_push(key="job_id", value=job_id)
96
+ print("Retrieved existing job with id: ", job_id)
97
+ return job_id
98
+ raise Exception("Failed to retrieve existing job information")
99
+
100
+ elif response.status_code == 200:
101
+ response_json = response.json()
102
+ job = response_json.get("job")
103
+ job_id = job.get("id")
104
+ ti.xcom_push(key="job_id", value=job_id)
105
+ print("Triggered new job with id: ", job_id)
106
+ return job_id
107
+ else:
108
+ print(f"Error: {response.status_code} - {response.text}")
109
+ raise Exception(f"Error: {response.status_code} - {response.text}")
110
+
111
+ except requests.exceptions.RequestException as e:
112
+ print(f"Request failed: {str(e)}")
113
+ raise Exception(f"Failed to trigger sync: {str(e)}")
114
+
115
+
116
+ def skip_if_already_run(i, ti=None, data_interval_end=None):
117
+ last_successful_sync_sec = ti.xcom_pull(
118
+ task_ids=f"airbyte.check_connections_status_{i}", key="last_successful_sync_sec"
119
+ )
120
+ d = datetime.utcfromtimestamp(int(last_successful_sync_sec))
121
+ last_sync = datetime(
122
+ year=d.year,
123
+ month=d.month,
124
+ day=d.day,
125
+ hour=d.hour,
126
+ minute=d.minute,
127
+ second=d.second,
128
+ tzinfo=pytz.UTC,
129
+ )
130
+ print("end of running date: ", data_interval_end)
131
+ print("last connection sync: ", last_sync)
132
+ diff_to_prev_run_in_hours = (data_interval_end - last_sync).total_seconds() / 3600
133
+ hours = timedelta(hours=diff_to_prev_run_in_hours)
134
+ print("Previous connection sync was: ", hours, " hours ago")
135
+ skip_next_task = hours <= timedelta(hours=1)
136
+ if skip_next_task:
137
+ return "airbyte.finish_airbyte"
138
+ else:
139
+ return f"airbyte.trigger_sync_{i}"
140
+
141
+
142
+ def skip_if_not_manual(i, ti=None):
143
+ schedule_type = ti.xcom_pull(
144
+ task_ids=f"airbyte.get_connections_info_{i}", key="schedule_type"
145
+ )
146
+ skip_next_task = schedule_type != "manual"
147
+ if skip_next_task:
148
+ print(f"Schedule type should be 'manual' to proceed, but it {schedule_type}!")
149
+ return "airbyte.finish_airbyte"
150
+ else:
151
+ return f"airbyte.check_connections_status_{i}"
152
+
153
+
154
+ def check_job(i, api, ti=None):
155
+ job_id = ti.xcom_pull(task_ids=f"airbyte.trigger_sync_{i}", key="job_id")
156
+ attempt = 1
157
+ while True:
158
+ url = f"{api}/api/v1/jobs/get"
159
+ headers = {"accept": "application/json"}
160
+ data = {"id": job_id}
161
+ response = requests.post(url, headers=headers, json=data)
162
+ if response.status_code == 200:
163
+ response_json = response.json()
164
+ job = response_json.get("job")
165
+ status = job.get("status")
166
+ print(f"Check: {attempt}, status: {status}")
167
+ if status in ("failed", 'cancelled', 'incomplete'):
168
+ raise Exception(f"Airbyte Job {job_id}: {status}.")
169
+ elif status == 'succeeded':
170
+ return True
171
+
172
+ else:
173
+ print(f"Error: {response.status_code} - {response.text}")
174
+ raise Exception(f"Error: {response.status_code} - {response.text}")
175
+
176
+ # Wait for 30 sec before making the next API call
177
+ time.sleep(30)
178
+ attempt += 1
179
+
180
+
181
+ class TaskBuilder:
182
+ def __init__(self, connection_ids) -> None:
183
+ self.airbyte_connection_ids = connection_ids
184
+
185
+ def build_tasks(self, connection_ids):
186
+
187
+ airbyte_conn = BaseHook.get_connection('data_replication_conn')
188
+
189
+ # Default host
190
+ default_host = 'data-replication-airbyte-webapp-svc.data-replication.svc.cluster.local'
191
+
192
+ host = default_host # Start with the default host
193
+
194
+ if airbyte_conn:
195
+ if airbyte_conn.host:
196
+ host = airbyte_conn.host
197
+ elif airbyte_conn.extra:
198
+ extra = airbyte_conn.extra_dejson
199
+ if 'host' in extra and extra['host']:
200
+ host = extra['host']
201
+
202
+ with TaskGroup(group_id="airbyte") as airbyte_group:
203
+ start_airbyte = EmptyOperator(task_id="start_airbyte")
204
+ finish_airbyte = EmptyOperator(
205
+ task_id="finish_airbyte", trigger_rule="none_failed_min_one_success"
206
+ )
207
+ for _, connection in enumerate(connection_ids):
208
+ index = connection.rsplit("-", 1)[-1]
209
+
210
+ task_get_connections_info = PythonOperator(
211
+ task_id=f"get_connections_info_{index}",
212
+ python_callable=get_con_info,
213
+ op_kwargs={"conn": connection, "api": host},
214
+ )
215
+
216
+ task_check_connection_schedule_type = BranchPythonOperator(
217
+ task_id=f"check_connection_schedule_type_{index}",
218
+ python_callable=skip_if_not_manual,
219
+ op_kwargs={"i": index},
220
+ )
221
+
222
+ task_check_connections_status = PythonOperator(
223
+ task_id=f"check_connections_status_{index}",
224
+ python_callable=get_con_status,
225
+ op_kwargs={"conn": connection, "api": host},
226
+ )
227
+
228
+ task_check_connections_last_sync = BranchPythonOperator(
229
+ task_id=f"check_connections_last_sync_{index}",
230
+ python_callable=skip_if_already_run,
231
+ op_kwargs={"i": index},
232
+ )
233
+
234
+ task_trigger_sync = PythonOperator(
235
+ task_id=f"trigger_sync_{index}",
236
+ python_callable=trigger_sync_job,
237
+ op_kwargs={"conn": connection, "api": host},
238
+ )
239
+
240
+ task_check_job = PythonOperator(
241
+ task_id=f"check_job_{index}",
242
+ python_callable=check_job,
243
+ op_kwargs={"i": index, "api": host},
244
+ )
245
+
246
+ (
247
+ start_airbyte
248
+ >> task_get_connections_info
249
+ >> task_check_connection_schedule_type
250
+ >> [finish_airbyte, task_check_connections_status]
251
+ )
252
+ (
253
+ task_check_connections_status
254
+ >> task_check_connections_last_sync
255
+ >> [finish_airbyte, task_trigger_sync]
256
+ )
257
+ task_trigger_sync >> task_check_job >> finish_airbyte
258
+
259
+ return airbyte_group
@@ -0,0 +1,3 @@
1
+ from fast2dev_dbt_runner.bash_operator.dbt_hook import *
2
+ from fast2dev_dbt_runner.bash_operator.dbt_operator import *
3
+ from fast2dev_dbt_runner.bash_operator.datawarehouse_secrets import *