fast2dev-dbt-runner 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fast2dev_dbt_runner/__init__.py +47 -0
- fast2dev_dbt_runner/_compat.py +41 -0
- fast2dev_dbt_runner/airbyte_task_group_builder.py +259 -0
- fast2dev_dbt_runner/bash_operator/__init__.py +3 -0
- fast2dev_dbt_runner/bash_operator/datawarehouse_secrets.py +490 -0
- fast2dev_dbt_runner/bash_operator/dbt_hook.py +615 -0
- fast2dev_dbt_runner/bash_operator/dbt_operator.py +248 -0
- fast2dev_dbt_runner/cached_manifest_loader.py +275 -0
- fast2dev_dbt_runner/dbt_manifest_parser_api_operator.py +452 -0
- fast2dev_dbt_runner/dbt_manifest_parser_bash_operator.py +361 -0
- fast2dev_dbt_runner/dbt_manifest_parser_gke_operator.py +401 -0
- fast2dev_dbt_runner/dbt_manifest_parser_k8s_operator.py +448 -0
- fast2dev_dbt_runner/utils.py +451 -0
- fast2dev_dbt_runner-0.0.1.dist-info/METADATA +398 -0
- fast2dev_dbt_runner-0.0.1.dist-info/RECORD +18 -0
- fast2dev_dbt_runner-0.0.1.dist-info/WHEEL +5 -0
- fast2dev_dbt_runner-0.0.1.dist-info/licenses/LICENSE +21 -0
- fast2dev_dbt_runner-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fast2dev-dbt-runner
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Airflow 3 operators that turn a dbt 1.12 manifest into DAGs for Fast2.Dev (k8s, bash, API, and GKE).
|
|
5
|
+
Author-email: M7XLab <support@fast2.dev>
|
|
6
|
+
Maintainer-email: "Fast2.Dev" <support@fast2.dev>
|
|
7
|
+
License: MIT
|
|
8
|
+
Project-URL: Homepage, https://fast2.dev
|
|
9
|
+
Project-URL: Documentation, https://fast2.dev
|
|
10
|
+
Project-URL: Repository, https://gitea.m7xlab.top/fast2dev/fast2dev-ddp
|
|
11
|
+
Project-URL: Bug Tracker, https://gitea.m7xlab.top/fast2dev/fast2dev-ddp/issues
|
|
12
|
+
Project-URL: Changelog, https://gitea.m7xlab.top/fast2dev/fast2dev-ddp/src/branch/main/services/dbt/workflow-runner/CHANGELOG.md
|
|
13
|
+
Keywords: airflow,airflow3,dbt,dbt-core,fast2dev,m7xlab,data-orchestration,kubernetes,gke,manifest-parser,workflow,etl,data-engineering
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Information Technology
|
|
17
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Topic :: Database
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
25
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
26
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
27
|
+
Classifier: Framework :: Apache Airflow
|
|
28
|
+
Requires-Python: <3.13,>=3.10
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Requires-Dist: kubernetes>=18.0.0
|
|
32
|
+
Requires-Dist: google-cloud-storage>=2.0.0
|
|
33
|
+
Requires-Dist: google-auth>=2.26.1
|
|
34
|
+
Requires-Dist: requests>=2.25.0
|
|
35
|
+
Requires-Dist: pyyaml>=5.4.0
|
|
36
|
+
Requires-Dist: jinja2>=3.0.0
|
|
37
|
+
Provides-Extra: airflow
|
|
38
|
+
Requires-Dist: apache-airflow<4,>=3.0.0; extra == "airflow"
|
|
39
|
+
Requires-Dist: apache-airflow-providers-cncf-kubernetes>=10.0.0; extra == "airflow"
|
|
40
|
+
Provides-Extra: dev
|
|
41
|
+
Requires-Dist: black>=21.0.0; extra == "dev"
|
|
42
|
+
Requires-Dist: build>=1.0.0; extra == "dev"
|
|
43
|
+
Requires-Dist: flake8>=3.8.0; extra == "dev"
|
|
44
|
+
Requires-Dist: mypy>=0.800; extra == "dev"
|
|
45
|
+
Requires-Dist: pre-commit>=2.15.0; extra == "dev"
|
|
46
|
+
Requires-Dist: pytest>=6.0.0; extra == "dev"
|
|
47
|
+
Requires-Dist: pytest-cov>=2.10.0; extra == "dev"
|
|
48
|
+
Requires-Dist: twine>=5.0.0; extra == "dev"
|
|
49
|
+
Requires-Dist: wheel>=0.37.0; extra == "dev"
|
|
50
|
+
Provides-Extra: docs
|
|
51
|
+
Requires-Dist: sphinx>=4.0.0; extra == "docs"
|
|
52
|
+
Requires-Dist: sphinx-rtd-theme>=1.0.0; extra == "docs"
|
|
53
|
+
Requires-Dist: myst-parser>=0.15.0; extra == "docs"
|
|
54
|
+
Dynamic: license-file
|
|
55
|
+
|
|
56
|
+
# fast2dev-dbt-runner
|
|
57
|
+
|
|
58
|
+
[](https://pypi.org/project/fast2dev-dbt-runner/)
|
|
59
|
+
[](https://www.python.org/downloads/)
|
|
60
|
+
[](https://opensource.org/licenses/MIT)
|
|
61
|
+
[](https://airflow.apache.org/)
|
|
62
|
+
[](https://docs.getdbt.com/)
|
|
63
|
+
|
|
64
|
+
Airflow 3 library that reads a dbt 1.12 `manifest.json` and builds DAGs for the [Fast2.Dev](https://fast2.dev) data platform. M7XLab publishes the package; worker images come from the private registry `registry.m7xlab.top/fast2dev/*`, not Docker Hub.
|
|
65
|
+
|
|
66
|
+
This is a **library**. There are no `console_scripts` entry points. Install it into the Airflow environment and import `fast2dev_dbt_runner`.
|
|
67
|
+
|
|
68
|
+
## What it does
|
|
69
|
+
|
|
70
|
+
Four execution backends, same manifest parser:
|
|
71
|
+
|
|
72
|
+
| Operator | Import alias | Best for | Worker image |
|
|
73
|
+
|----------|--------------|----------|--------------|
|
|
74
|
+
| `k8s` | `DbtManifestParserK8sOperator` | Daily jobs, high concurrency, lowest cost | `registry.m7xlab.top/fast2dev/dbt-workflow-core` |
|
|
75
|
+
| `bash` | `DbtManifestParserBashOperator` | Medium projects on Airflow workers | Airflow worker image |
|
|
76
|
+
| `api` | `DbtManifestParserApiOperator` | Low-latency runs against a pre-warmed dbt API | Fast2.Dev dbt api-server |
|
|
77
|
+
| `gke` | `DbtManifestParserGkeOperator` | Isolated external-client clusters | `registry.m7xlab.top/fast2dev/dbt-workflow-core` |
|
|
78
|
+
|
|
79
|
+
DAG templates in the Fast2.Dev platform import the parser classes from the operator modules directly (for example `from fast2dev_dbt_runner.dbt_manifest_parser_k8s_operator import DbtManifestParser`). The aliases above are the public package-level names.
|
|
80
|
+
|
|
81
|
+
## Requirements
|
|
82
|
+
|
|
83
|
+
- Python 3.10–3.12
|
|
84
|
+
- Apache Airflow 3.x (install the `[airflow]` extra, or rely on the platform image)
|
|
85
|
+
- dbt Core 1.12 manifests (schema v12)
|
|
86
|
+
- Private registry access for k8s/GKE images (`registry.m7xlab.top`)
|
|
87
|
+
|
|
88
|
+
## Installation
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
pip install fast2dev-dbt-runner
|
|
92
|
+
|
|
93
|
+
# Airflow 3 + Kubernetes provider
|
|
94
|
+
pip install fast2dev-dbt-runner[airflow]
|
|
95
|
+
|
|
96
|
+
# Local development (tests, linters, twine)
|
|
97
|
+
pip install fast2dev-dbt-runner[dev]
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
From the Fast2.Dev monorepo (this package is not a standalone GitHub repository):
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
git clone git@gitea.m7xlab.top:fast2dev/fast2dev-ddp.git
|
|
104
|
+
cd fast2dev-ddp/services/dbt/workflow-runner
|
|
105
|
+
pip install -e ".[dev]"
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Publishing to pypi.org is documented in [docs/publishing.md](docs/publishing.md). Do not commit a PyPI token or a `.pypirc` file.
|
|
109
|
+
|
|
110
|
+
## Usage
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from airflow.sdk import DAG
|
|
114
|
+
from fast2dev_dbt_runner import DbtManifestParserK8sOperator
|
|
115
|
+
|
|
116
|
+
with DAG(
|
|
117
|
+
dag_id="analytics_dbt",
|
|
118
|
+
schedule="@daily",
|
|
119
|
+
catchup=False,
|
|
120
|
+
) as dag:
|
|
121
|
+
parser = DbtManifestParserK8sOperator(...)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Platform-generated DAGs live in `services/cicd/runner` and already import `fast2dev_dbt_runner`. Configure behavior with Airflow Variables, not by editing this library.
|
|
125
|
+
|
|
126
|
+
## Airflow Variable Reference
|
|
127
|
+
|
|
128
|
+
All variables are read from Airflow Variables at DAG load time. Defaults shown in parentheses.
|
|
129
|
+
|
|
130
|
+
### Infrastructure & Identity
|
|
131
|
+
|
|
132
|
+
| Variable | Default | Description |
|
|
133
|
+
|----------|---------|-------------|
|
|
134
|
+
| `PROJECT_ID` | required | Cloud project identifier |
|
|
135
|
+
| `DBT_PROJECT_NAME` | required | dbt project name (used as DAG ID prefix) |
|
|
136
|
+
| `NAMESPACE` | — | Kubernetes namespace (k8s/GKE operators) |
|
|
137
|
+
| `DAG_OWNER` | `fast2.dev` | Airflow DAG owner |
|
|
138
|
+
| `DAG_START_DATE` | `days_ago(1)` | DAG start date expression |
|
|
139
|
+
| `DAG_SCHEDULE_INTERVAL` | `@once` | Cron expression or preset (`@daily`, `@hourly`, etc.) |
|
|
140
|
+
| `GIT_BRANCH` | — | Git branch to checkout on the worker before running dbt |
|
|
141
|
+
| `IMAGE` | platform default | dbt worker image, typically `registry.m7xlab.top/fast2dev/dbt-workflow-core:<tag>` |
|
|
142
|
+
|
|
143
|
+
### Model Execution Control
|
|
144
|
+
|
|
145
|
+
These three flags follow the same pattern: `is_in_manifest` → `DBT_X` → `DBT_X_SHARDING`.
|
|
146
|
+
|
|
147
|
+
| Variable | Default | Description |
|
|
148
|
+
|----------|---------|-------------|
|
|
149
|
+
| `DBT_MODEL` | `True` | Enable/disable model (`dbt run`) execution entirely |
|
|
150
|
+
| `DBT_MODEL_SHARDING` | `True` | `True` = one Airflow task per model with full dependency lineage; `False` = single batch task running `dbt run --select "model1 model2 ..."` |
|
|
151
|
+
|
|
152
|
+
### Seed Execution Control
|
|
153
|
+
|
|
154
|
+
| Variable | Default | Description |
|
|
155
|
+
|----------|---------|-------------|
|
|
156
|
+
| `DBT_SEED` | `False` | Enable/disable seed (`dbt seed`) execution |
|
|
157
|
+
| `DBT_SEED_SHARDING` | `True` | `True` = one task per seed file; `False` = single batch task with `--select` |
|
|
158
|
+
|
|
159
|
+
### Snapshot Execution Control
|
|
160
|
+
|
|
161
|
+
| Variable | Default | Description |
|
|
162
|
+
|----------|---------|-------------|
|
|
163
|
+
| `DBT_SNAPSHOT` | `False` | Enable/disable snapshot (`dbt snapshot`) execution |
|
|
164
|
+
| `DBT_SNAPSHOT_SHARDING` | `True` | `True` = one task per snapshot; `False` = single batch task with `--select` |
|
|
165
|
+
| `DBT_MODEL_DEPENDS_ON_SNAPSHOT` | `False` | When `True`, a `model -> snapshot -> model` chain (a snapshot a model depends on via `ref()`) runs in correct order within a single DAG. **Sharded** (`DBT_MODEL_SHARDING=True`): the snapshot (and its tests) is woven into the model task group as task-level dependencies. **Batch** (`DBT_MODEL_SHARDING=False`): the model-run + snapshot batches are replaced by a single `dbt build --select "<models> <snapshots>"`, which orders models, snapshots, and tests by lineage in one process. No-op when no model actually depends on a snapshot. Supported across all operators (bash, k8s, GKE, API). |
|
|
166
|
+
|
|
167
|
+
### Source Freshness Control
|
|
168
|
+
|
|
169
|
+
| Variable | Default | Description |
|
|
170
|
+
|----------|---------|-------------|
|
|
171
|
+
| `DBT_SOURCE` | `True` | Enable/disable source freshness (`dbt source freshness`) checks |
|
|
172
|
+
| `DBT_SOURCE_SHARDING` | `True` | `True` = one task per source; `False` = single batch task with `--select` |
|
|
173
|
+
|
|
174
|
+
### Pipeline Steps
|
|
175
|
+
|
|
176
|
+
| Variable | Default | Description |
|
|
177
|
+
|----------|---------|-------------|
|
|
178
|
+
| `DBT_DEPS` | `True` | Run `dbt deps` at DAG start to install packages. Set `False` when packages are vendored in the repo |
|
|
179
|
+
| `DATA_QUALITY` | `False` | Enable re_data / data quality task at end of DAG |
|
|
180
|
+
| `DEBUG` | `False` | Run `dbt debug` at DAG start to verify connection |
|
|
181
|
+
|
|
182
|
+
### Filtering & Selection
|
|
183
|
+
|
|
184
|
+
| Variable | Default | Description |
|
|
185
|
+
|----------|---------|-------------|
|
|
186
|
+
| `DBT_TAGS` | — | Comma-separated list of dbt tags. Only nodes tagged with **all** listed tags are included. Example: `tag1,tag2` |
|
|
187
|
+
|
|
188
|
+
### Full Refresh
|
|
189
|
+
|
|
190
|
+
| Variable | Default | Description |
|
|
191
|
+
|----------|---------|-------------|
|
|
192
|
+
| `FULL_REFRESH` | `False` | Run models with `dbt run --full-refresh` (rebuilds incrementals from scratch) |
|
|
193
|
+
| `FULL_REFRESH_MODEL_NAME` | — | Comma-separated list of specific model names to full-refresh (others run normally) |
|
|
194
|
+
|
|
195
|
+
### E2E / Testing Modes
|
|
196
|
+
|
|
197
|
+
| Variable | Default | Description |
|
|
198
|
+
|----------|---------|-------------|
|
|
199
|
+
| `E2E_MODE_EMPTY` | `False` | Append `--empty` to `dbt run` (creates empty tables without processing data, for E2E schema validation) |
|
|
200
|
+
|
|
201
|
+
### Monitoring & Logging
|
|
202
|
+
|
|
203
|
+
| Variable | Default | Description |
|
|
204
|
+
|----------|---------|-------------|
|
|
205
|
+
| `MODEL_DEBUG_LOG` | `False` | Log compiled SQL for failed model tasks (appends compiled code to Airflow task logs) |
|
|
206
|
+
| `DATAHUB_ENABLED` | `False` | Enable DataHub metadata push after DAG run |
|
|
207
|
+
|
|
208
|
+
### Manifest Cache (Environment Variables)
|
|
209
|
+
|
|
210
|
+
These are set as environment variables on the Airflow workers, not as Airflow Variables:
|
|
211
|
+
|
|
212
|
+
| Variable | Default | Description |
|
|
213
|
+
|----------|---------|-------------|
|
|
214
|
+
| `AIRFLOW__CORE__MANIFEST_CACHE_ENABLED` | `True` | Enable manifest caching (reduces DAG parse time by 99%+ for unchanged manifests) |
|
|
215
|
+
| `AIRFLOW__CORE__MANIFEST_CACHE_DEBUG` | `False` | Log cache hit/miss details |
|
|
216
|
+
| `AIRFLOW__CORE__MANIFEST_CACHE_MAX_SIZE` | `50` | Maximum number of manifests to keep in the module-level cache |
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
## Sharding Explained
|
|
221
|
+
|
|
222
|
+
### Sharding = True (default) — Full Lineage
|
|
223
|
+
|
|
224
|
+
Each model/seed/snapshot/source becomes its own Airflow task. Airflow builds the full dependency graph from the dbt manifest, so tasks execute in dependency order and you can retry individual failed nodes.
|
|
225
|
+
|
|
226
|
+
```
|
|
227
|
+
seed_customers → model_stg_customers → model_orders → model_revenue
|
|
228
|
+
↗
|
|
229
|
+
model_stg_orders ───
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
**Use when**: You need visibility into individual model failures, want to retry a single model, or have long-running models that benefit from parallelism.
|
|
233
|
+
|
|
234
|
+
### Sharding = False — Batch Mode
|
|
235
|
+
|
|
236
|
+
All tag-filtered models are collected from the manifest and passed in a single `dbt run --select "model1 model2 ..."` command, running as one Airflow task. Tag filtering (`DBT_TAGS`) is fully respected — only manifested, tag-matching nodes are included.
|
|
237
|
+
|
|
238
|
+
```
|
|
239
|
+
single_task: dbt run --select "stg_customers stg_orders dim_revenue"
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
**Use when**: You have many small models, want simpler DAGs with fewer tasks, or Airflow overhead per-task is significant.
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
## Configuration Examples
|
|
247
|
+
|
|
248
|
+
### Standard Daily Pipeline (K8S)
|
|
249
|
+
|
|
250
|
+
```python
|
|
251
|
+
# Airflow Variables
|
|
252
|
+
{
|
|
253
|
+
"PROJECT_ID": "my-gcp-project",
|
|
254
|
+
"DBT_PROJECT_NAME": "analytics",
|
|
255
|
+
"DAG_SCHEDULE_INTERVAL": "@daily",
|
|
256
|
+
"DBT_DEPS": "True",
|
|
257
|
+
"DBT_SOURCE": "True",
|
|
258
|
+
"DBT_SOURCE_SHARDING": "True",
|
|
259
|
+
"DBT_SEED": "False",
|
|
260
|
+
"DBT_MODEL": "True",
|
|
261
|
+
"DBT_MODEL_SHARDING": "True",
|
|
262
|
+
"DBT_SNAPSHOT": "False",
|
|
263
|
+
"DATA_QUALITY": "True"
|
|
264
|
+
}
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
### Batch Mode (Many Small Models, Low Overhead)
|
|
268
|
+
|
|
269
|
+
```python
|
|
270
|
+
# All resource types run as single batch tasks — fewer Airflow tasks, simpler DAG
|
|
271
|
+
{
|
|
272
|
+
"DBT_MODEL": "True",
|
|
273
|
+
"DBT_MODEL_SHARDING": "False", # dbt run --select "model1 model2 ..."
|
|
274
|
+
"DBT_SEED": "True",
|
|
275
|
+
"DBT_SEED_SHARDING": "False", # dbt seed --select "seed1 seed2 ..."
|
|
276
|
+
"DBT_SOURCE": "True",
|
|
277
|
+
"DBT_SOURCE_SHARDING": "False", # dbt source freshness --select "src1 src2 ..."
|
|
278
|
+
"DBT_SNAPSHOT": "True",
|
|
279
|
+
"DBT_SNAPSHOT_SHARDING": "False" # dbt snapshot --select "snap1 snap2 ..."
|
|
280
|
+
}
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
### Tag-Filtered Pipeline
|
|
284
|
+
|
|
285
|
+
```python
|
|
286
|
+
# Only run nodes tagged with both "marketing" and "daily"
|
|
287
|
+
{
|
|
288
|
+
"DBT_TAGS": "marketing,daily",
|
|
289
|
+
"DBT_MODEL": "True",
|
|
290
|
+
"DBT_MODEL_SHARDING": "True"
|
|
291
|
+
}
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
### Full Refresh Specific Models
|
|
295
|
+
|
|
296
|
+
```python
|
|
297
|
+
# Full refresh only two models; others run normally
|
|
298
|
+
{
|
|
299
|
+
"FULL_REFRESH": "False",
|
|
300
|
+
"FULL_REFRESH_MODEL_NAME": "dim_customers,fct_orders"
|
|
301
|
+
}
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
### Full Refresh All Models
|
|
305
|
+
|
|
306
|
+
```python
|
|
307
|
+
{
|
|
308
|
+
"FULL_REFRESH": "True"
|
|
309
|
+
}
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
### E2E Schema Validation
|
|
313
|
+
|
|
314
|
+
```python
|
|
315
|
+
# Creates empty tables (no data) to validate schema changes end-to-end
|
|
316
|
+
{
|
|
317
|
+
"E2E_MODE_EMPTY": "True",
|
|
318
|
+
"DBT_MODEL": "True",
|
|
319
|
+
"DBT_MODEL_SHARDING": "False"
|
|
320
|
+
}
|
|
321
|
+
```
|
|
322
|
+
|
|
323
|
+
### Skip Package Installation (Vendored Packages)
|
|
324
|
+
|
|
325
|
+
```python
|
|
326
|
+
# Packages are committed to the repo — skip dbt deps for faster, more reliable runs
|
|
327
|
+
{
|
|
328
|
+
"DBT_DEPS": "False"
|
|
329
|
+
}
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
### High-Performance Real-Time Pipeline (API Operator)
|
|
333
|
+
|
|
334
|
+
```python
|
|
335
|
+
{
|
|
336
|
+
"PROJECT_ID": "my-gcp-project",
|
|
337
|
+
"DBT_PROJECT_NAME": "realtime_analytics",
|
|
338
|
+
"DAG_SCHEDULE_INTERVAL": "*/15 * * * *",
|
|
339
|
+
"DBT_DEPS": "False",
|
|
340
|
+
"DBT_MODEL": "True",
|
|
341
|
+
"DBT_MODEL_SHARDING": "False", # batch mode for speed
|
|
342
|
+
"DBT_TAGS": "realtime",
|
|
343
|
+
"MODEL_DEBUG_LOG": "True"
|
|
344
|
+
}
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
---
|
|
348
|
+
|
|
349
|
+
## Architecture
|
|
350
|
+
|
|
351
|
+
### Execution Flow per DAG
|
|
352
|
+
|
|
353
|
+
```
|
|
354
|
+
[Airbyte sync] (optional)
|
|
355
|
+
↓
|
|
356
|
+
[dbt deps] (if DBT_DEPS=True)
|
|
357
|
+
↓
|
|
358
|
+
[dbt debug] (if DEBUG=True)
|
|
359
|
+
↓
|
|
360
|
+
[show_input_data]
|
|
361
|
+
↓
|
|
362
|
+
[dbt source freshness] (if DBT_SOURCE=True)
|
|
363
|
+
↓
|
|
364
|
+
[dbt seed] (if DBT_SEED=True)
|
|
365
|
+
↓
|
|
366
|
+
[dbt run] (if DBT_MODEL=True)
|
|
367
|
+
↓
|
|
368
|
+
[dbt snapshot] (if DBT_SNAPSHOT=True)
|
|
369
|
+
↓
|
|
370
|
+
[re_data / quality] (if DATA_QUALITY=True)
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
### Manifest Caching
|
|
374
|
+
|
|
375
|
+
The manifest caching system reduces DAG import time by 99%+ for unchanged manifests:
|
|
376
|
+
|
|
377
|
+
- **Before caching**: ~2–4 seconds per manifest parse, ~480 parses/hour with 2 schedulers
|
|
378
|
+
- **After caching**: <10ms for cache hits, only 5–10 cache misses/hour (on actual manifest changes)
|
|
379
|
+
- Cache keys include: file MD5 hash + `DBT_TAGS` + ancestor/descendant flags
|
|
380
|
+
- Cache is process-local (not shared across pod restarts); first parse after restart is always a cache miss
|
|
381
|
+
|
|
382
|
+
---
|
|
383
|
+
|
|
384
|
+
## Development
|
|
385
|
+
|
|
386
|
+
```bash
|
|
387
|
+
pytest tests/
|
|
388
|
+
pytest tests/ --cov=fast2dev_dbt_runner --cov-report=term-missing
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) and [docs/publishing.md](docs/publishing.md) for release steps. Version **0.0.1** is the first public PyPI name under M7XLab / Fast2.Dev. Prior calendar versions (for example `2026.1.0.9`) were internal.
|
|
392
|
+
|
|
393
|
+
## Support
|
|
394
|
+
|
|
395
|
+
- **Homepage**: [fast2.dev](https://fast2.dev)
|
|
396
|
+
- **Source**: [gitea.m7xlab.top/fast2dev/fast2dev-ddp](https://gitea.m7xlab.top/fast2dev/fast2dev-ddp) (`services/dbt/workflow-runner`)
|
|
397
|
+
- **Email**: support@fast2.dev
|
|
398
|
+
- **Changelog**: [CHANGELOG.md](CHANGELOG.md)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
fast2dev_dbt_runner/__init__.py,sha256=b9fZzpK2W_Y48IiO_pp_kQCdYXVGMsySkvEjfkZ7IrY,1443
|
|
2
|
+
fast2dev_dbt_runner/_compat.py,sha256=3zRkZQYGtXBAlQAW1yKR38QE2B4qF-IPhOUcGerIZMc,1129
|
|
3
|
+
fast2dev_dbt_runner/airbyte_task_group_builder.py,sha256=9uZ-6xMRQPwMvfhuYnZua0B5Ygdr5BcvzN2ia05SvqE,9831
|
|
4
|
+
fast2dev_dbt_runner/cached_manifest_loader.py,sha256=JBZfogPbcntcmVNnglWr-8rI4KsbuiMBfYThOZ1jBGU,9544
|
|
5
|
+
fast2dev_dbt_runner/dbt_manifest_parser_api_operator.py,sha256=f1vtdGqzGabaHrMZOcgOHIEQ-W8OLxQnDkhpps-RPVg,21185
|
|
6
|
+
fast2dev_dbt_runner/dbt_manifest_parser_bash_operator.py,sha256=i1_L1IcNtC9vjagHL6UXm_8Y_wx_PqRoTOHFkFBDCwM,16803
|
|
7
|
+
fast2dev_dbt_runner/dbt_manifest_parser_gke_operator.py,sha256=_BX3BTWWzfRLXXQnk7XNhCoQLL4UXo_zMnS7iHsEgJg,19722
|
|
8
|
+
fast2dev_dbt_runner/dbt_manifest_parser_k8s_operator.py,sha256=1UQ-9ddSPTbt20uIwictzct3Kl1GLbkMeP4DR0TxgYw,21663
|
|
9
|
+
fast2dev_dbt_runner/utils.py,sha256=Z-MHKRK72eOlcdNWxYfbmDmDu3xewrCuDOD4818Siec,18577
|
|
10
|
+
fast2dev_dbt_runner/bash_operator/__init__.py,sha256=irc43XUbjin8tT7YZdv07PVexEad1FjMdZArvl6d_54,188
|
|
11
|
+
fast2dev_dbt_runner/bash_operator/datawarehouse_secrets.py,sha256=th2Da-YcpkQAiE7iCwhojt54GWLOYvLj5apZ2eTsVQ4,21527
|
|
12
|
+
fast2dev_dbt_runner/bash_operator/dbt_hook.py,sha256=vThFv74qXwCjSU9EplfPITa0tKURMEe4e80arkO6iRk,28189
|
|
13
|
+
fast2dev_dbt_runner/bash_operator/dbt_operator.py,sha256=BLfMb_T_ZK-2UM0_KPhMqcd8s95S7dGlFeM2jiQZWaw,10612
|
|
14
|
+
fast2dev_dbt_runner-0.0.1.dist-info/licenses/LICENSE,sha256=b_EVN97jYjtkllX7Mln0Anufu6kfFbpAbddGH894ueY,1063
|
|
15
|
+
fast2dev_dbt_runner-0.0.1.dist-info/METADATA,sha256=rtwPQ29j2Da23dwEgtyihSvMtTFQAaUce1O686okF18,15425
|
|
16
|
+
fast2dev_dbt_runner-0.0.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
17
|
+
fast2dev_dbt_runner-0.0.1.dist-info/top_level.txt,sha256=vlC0zj40GP03j45qip2HDV7bI5XaeAta8IhfqbW73Nw,20
|
|
18
|
+
fast2dev_dbt_runner-0.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 M7XLab
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fast2dev_dbt_runner
|