dlt-firebolt 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dlt_firebolt-0.1.0/.dlt/config.toml +3 -0
- dlt_firebolt-0.1.0/.dlt/secrets.toml.example +20 -0
- dlt_firebolt-0.1.0/.env.example +17 -0
- dlt_firebolt-0.1.0/.github/workflows/ci.yml +27 -0
- dlt_firebolt-0.1.0/.github/workflows/publish.yml +39 -0
- dlt_firebolt-0.1.0/.gitignore +12 -0
- dlt_firebolt-0.1.0/LICENSE +18 -0
- dlt_firebolt-0.1.0/PKG-INFO +194 -0
- dlt_firebolt-0.1.0/README.md +170 -0
- dlt_firebolt-0.1.0/SPIKE.md +55 -0
- dlt_firebolt-0.1.0/check_firebolt_env.py +123 -0
- dlt_firebolt-0.1.0/firebolt_dest/README.md +102 -0
- dlt_firebolt-0.1.0/firebolt_dest/__init__.py +13 -0
- dlt_firebolt-0.1.0/firebolt_dest/client.py +154 -0
- dlt_firebolt-0.1.0/firebolt_dest/configuration.py +125 -0
- dlt_firebolt-0.1.0/firebolt_dest/copy_sql.py +29 -0
- dlt_firebolt-0.1.0/firebolt_dest/factory.py +115 -0
- dlt_firebolt-0.1.0/firebolt_dest/sql_client.py +181 -0
- dlt_firebolt-0.1.0/phase1_copy.sql +11 -0
- dlt_firebolt-0.1.0/phase1_hubspot_to_s3.py +173 -0
- dlt_firebolt-0.1.0/phase2_dialect_smoke.py +149 -0
- dlt_firebolt-0.1.0/phase3_hubspot_to_firebolt.py +100 -0
- dlt_firebolt-0.1.0/phase4_dispositions.py +74 -0
- dlt_firebolt-0.1.0/phase5_nested.py +71 -0
- dlt_firebolt-0.1.0/pyproject.toml +43 -0
- dlt_firebolt-0.1.0/requirements-dev.txt +1 -0
- dlt_firebolt-0.1.0/requirements.txt +4 -0
- dlt_firebolt-0.1.0/tests/conftest.py +13 -0
- dlt_firebolt-0.1.0/tests/test_capabilities.py +11 -0
- dlt_firebolt-0.1.0/tests/test_copy_sql.py +24 -0
- dlt_firebolt-0.1.0/tests/test_credentials.py +19 -0
- dlt_firebolt-0.1.0/tests/test_integration.py +45 -0
- dlt_firebolt-0.1.0/tests/test_merge_sql.py +114 -0
- dlt_firebolt-0.1.0/tests/test_sql_client.py +57 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Copy to .dlt/secrets.toml (gitignored). Do not commit real credentials.
|
|
2
|
+
|
|
3
|
+
[destination.firebolt]
|
|
4
|
+
# Option A: connection string (recommended)
|
|
5
|
+
# credentials = "firebolt://CLIENT_ID:CLIENT_SECRET@DATABASE/ENGINE?account_name=ACCOUNT"
|
|
6
|
+
|
|
7
|
+
s3_location_name = "your_location_name"
|
|
8
|
+
s3_prefix = "dlt-landing"
|
|
9
|
+
|
|
10
|
+
# Option B: structured credentials
|
|
11
|
+
# Firebolt maps to SQLAlchemy URL as host=DATABASE, database=ENGINE (see firebolt_url_from_env).
|
|
12
|
+
[destination.firebolt.credentials]
|
|
13
|
+
host = "YOUR_DATABASE"
|
|
14
|
+
database = "YOUR_ENGINE"
|
|
15
|
+
username = "YOUR_CLIENT_ID"
|
|
16
|
+
password = "YOUR_CLIENT_SECRET"
|
|
17
|
+
account_name = "YOUR_ACCOUNT_NAME"
|
|
18
|
+
|
|
19
|
+
[destination.filesystem]
|
|
20
|
+
bucket_url = "s3://YOUR_BUCKET/dlt-landing/dlt/staging"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Firebolt service account (Settings → Service accounts)
|
|
2
|
+
FIREBOLT_CLIENT_ID=
|
|
3
|
+
FIREBOLT_CLIENT_SECRET=
|
|
4
|
+
FIREBOLT_ACCOUNT_NAME=
|
|
5
|
+
FIREBOLT_DATABASE=
|
|
6
|
+
FIREBOLT_ENGINE=
|
|
7
|
+
|
|
8
|
+
# S3 staging — must match the name from your Firebolt CREATE LOCATION statement
|
|
9
|
+
# (e.g. CREATE LOCATION "sprinto_s3" WITH SOURCE='CLOUD_STORAGE' URL='s3://...')
|
|
10
|
+
FIREBOLT_S3_LOCATION_NAME=sprinto_s3
|
|
11
|
+
S3_BUCKET=
|
|
12
|
+
S3_PREFIX=dlt-landing
|
|
13
|
+
|
|
14
|
+
# Optional: HubSpot demo scripts only (phase1 / phase3)
|
|
15
|
+
HUBSPOT_ACCESS_TOKEN=
|
|
16
|
+
|
|
17
|
+
# AWS credentials for S3 staging (e.g. export AWS_PROFILE=your-profile)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
matrix:
|
|
13
|
+
python-version: ["3.10", "3.12"]
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: ${{ matrix.python-version }}
|
|
20
|
+
|
|
21
|
+
- name: Install package and test deps
|
|
22
|
+
run: |
|
|
23
|
+
python -m pip install --upgrade pip
|
|
24
|
+
pip install -e ".[dev]"
|
|
25
|
+
|
|
26
|
+
- name: Unit tests
|
|
27
|
+
run: pytest -q -m "not integration"
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
# Mirrors firebolt-db Python SDK auth: TWINE_* from PYPI_USERNAME / PYPI_PASSWORD.
|
|
4
|
+
# Prerequisite: add those secrets to firebolt-analytics/dlt-firebolt (org secrets
|
|
5
|
+
# do not cross from firebolt-db). Bump version in pyproject.toml before release.
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
release:
|
|
9
|
+
types: [published]
|
|
10
|
+
workflow_dispatch:
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
publish:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: "3.12"
|
|
23
|
+
|
|
24
|
+
- name: Install build tooling
|
|
25
|
+
run: |
|
|
26
|
+
python -m pip install --upgrade pip
|
|
27
|
+
pip install -e ".[dev]" build twine
|
|
28
|
+
|
|
29
|
+
- name: Unit tests
|
|
30
|
+
run: pytest -q -m "not integration"
|
|
31
|
+
|
|
32
|
+
- name: Build
|
|
33
|
+
run: python -m build
|
|
34
|
+
|
|
35
|
+
- name: Publish
|
|
36
|
+
env:
|
|
37
|
+
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
|
|
38
|
+
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
|
|
39
|
+
run: twine upload dist/*
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
Copyright 2026 Firebolt Analytics Inc.
|
|
7
|
+
|
|
8
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
9
|
+
you may not use this file except in compliance with the License.
|
|
10
|
+
You may obtain a copy of the License at
|
|
11
|
+
|
|
12
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
13
|
+
|
|
14
|
+
Unless required by applicable law or agreed to in writing, software
|
|
15
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
16
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
17
|
+
See the License for the specific language governing permissions and
|
|
18
|
+
limitations under the License.
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dlt-firebolt
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: dlt destination for Firebolt (staged Parquet + COPY INTO)
|
|
5
|
+
Project-URL: Homepage, https://github.com/firebolt-analytics/dlt-firebolt
|
|
6
|
+
Project-URL: Repository, https://github.com/firebolt-analytics/dlt-firebolt
|
|
7
|
+
Author: Firebolt Analytics
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: data-pipeline,dlt,etl,firebolt
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Database
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Requires-Dist: dlt[parquet,s3,sqlalchemy]>=1.0.0
|
|
18
|
+
Requires-Dist: firebolt-sqlalchemy
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
21
|
+
Requires-Dist: python-dotenv; extra == 'dev'
|
|
22
|
+
Requires-Dist: requests; extra == 'dev'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# dlt-firebolt
|
|
26
|
+
|
|
27
|
+
Prototype [dlt](https://dlthub.com/) destination for [Firebolt](https://www.firebolt.io/).
|
|
28
|
+
|
|
29
|
+
Loads dlt pipelines into Firebolt using **filesystem staging (Parquet on S3) + `COPY INTO`**, the same pattern as dlt's Snowflake and Redshift destinations.
|
|
30
|
+
|
|
31
|
+
## Status
|
|
32
|
+
|
|
33
|
+
Spike complete. Hardening done; packaging and upstream prep in progress.
|
|
34
|
+
|
|
35
|
+
| Phase | What it proved |
|
|
36
|
+
|-------|----------------|
|
|
37
|
+
| 1 | dlt → S3 Parquet → manual COPY INTO |
|
|
38
|
+
| 2 | Generic `sqlalchemy` destination is not viable on Firebolt |
|
|
39
|
+
| 3 | Native `destination="firebolt"` end-to-end |
|
|
40
|
+
| 4 | Append / merge / replace disposition scripts |
|
|
41
|
+
|
|
42
|
+
See [SPIKE.md](SPIKE.md) for spike notes.
|
|
43
|
+
|
|
44
|
+
## License
|
|
45
|
+
|
|
46
|
+
Apache License 2.0 — see [LICENSE](LICENSE).
|
|
47
|
+
|
|
48
|
+
## Install
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
python -m venv .venv
|
|
52
|
+
source .venv/bin/activate
|
|
53
|
+
pip install -e ".[dev]"
|
|
54
|
+
cp .env.example .env # fill in Firebolt + S3 creds
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Or install dependencies only (no editable package):
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pip install -r requirements.txt
|
|
61
|
+
pip install -r requirements-dev.txt
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Quick start (Phase 3 demo)
|
|
65
|
+
|
|
66
|
+
Requires:
|
|
67
|
+
|
|
68
|
+
- Firebolt `CREATE LOCATION` for your S3 bucket — set `FIREBOLT_S3_LOCATION_NAME` to the location name (e.g. `sprinto_s3`)
|
|
69
|
+
- HubSpot private app token in `.env` (demo only)
|
|
70
|
+
- AWS credentials for S3 staging
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
export AWS_PROFILE=your-profile
|
|
74
|
+
python phase3_hubspot_to_firebolt.py
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Optional: copy `.dlt/secrets.toml.example` to `.dlt/secrets.toml` and run with `DLT_USE_SECRETS=1`.
|
|
78
|
+
|
|
79
|
+
Before running demos, validate credentials:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
python check_firebolt_env.py
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### Disposition checks (Phase 4)
|
|
86
|
+
|
|
87
|
+
Run each command separately (do not paste inline comments):
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
python phase4_dispositions.py --mode merge
|
|
91
|
+
python phase4_dispositions.py --mode append
|
|
92
|
+
python phase4_dispositions.py --mode append
|
|
93
|
+
python phase4_dispositions.py --mode replace
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
For append, run the command twice and confirm the row count grows.
|
|
97
|
+
|
|
98
|
+
Verify in Firebolt (default dataset `demo`):
|
|
99
|
+
|
|
100
|
+
```sql
|
|
101
|
+
SELECT COUNT(*) FROM demo_hubspot_contacts;
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Usage in a dlt pipeline
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
import sys
|
|
108
|
+
from pathlib import Path
|
|
109
|
+
|
|
110
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
111
|
+
|
|
112
|
+
import dlt
|
|
113
|
+
from firebolt_dest.configuration import make_firebolt_pipeline
|
|
114
|
+
|
|
115
|
+
pipeline = make_firebolt_pipeline(
|
|
116
|
+
pipeline_name="my_pipeline",
|
|
117
|
+
dataset_name="my_dataset",
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
pipeline.run(my_resource(), loader_file_format="parquet")
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Or with `.dlt/secrets.toml`:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
pipeline = make_firebolt_pipeline(
|
|
127
|
+
pipeline_name="my_pipeline",
|
|
128
|
+
dataset_name="my_dataset",
|
|
129
|
+
from_secrets=True,
|
|
130
|
+
)
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Tables land as `{dataset}_{table}` (e.g. `my_dataset_orders`).
|
|
134
|
+
|
|
135
|
+
Connection details from environment variables — see [.env.example](.env.example) — or from `.dlt/secrets.toml` — see [.dlt/secrets.toml.example](.dlt/secrets.toml.example).
|
|
136
|
+
|
|
137
|
+
## Layout
|
|
138
|
+
|
|
139
|
+
```
|
|
140
|
+
firebolt_dest/ # destination implementation (fork Redshift COPY pattern)
|
|
141
|
+
factory.py # registers destination="firebolt"
|
|
142
|
+
client.py # COPY load jobs
|
|
143
|
+
sql_client.py # Firebolt SQLAlchemy client
|
|
144
|
+
copy_sql.py # COPY INTO SQL generation
|
|
145
|
+
configuration.py # credentials + S3 location config
|
|
146
|
+
phase1_*.py # spike: dlt → S3 only
|
|
147
|
+
phase2_*.py # spike: dialect smoke test
|
|
148
|
+
phase3_*.py # spike: full native destination demo
|
|
149
|
+
phase4_*.py # append / merge / replace disposition checks
|
|
150
|
+
.dlt/config.toml # non-sensitive dlt defaults (parquet loader)
|
|
151
|
+
.dlt/secrets.toml.example
|
|
152
|
+
tests/ # unit tests (no Firebolt connection)
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
## Configuration
|
|
156
|
+
|
|
157
|
+
| Variable | Required | Description |
|
|
158
|
+
|----------|----------|-------------|
|
|
159
|
+
| `FIREBOLT_CLIENT_ID` | yes | Service account client ID |
|
|
160
|
+
| `FIREBOLT_CLIENT_SECRET` | yes | Service account secret |
|
|
161
|
+
| `FIREBOLT_ACCOUNT_NAME` | yes | Firebolt account name |
|
|
162
|
+
| `FIREBOLT_DATABASE` | yes | Target database |
|
|
163
|
+
| `FIREBOLT_ENGINE` | yes | Engine name |
|
|
164
|
+
| `FIREBOLT_S3_LOCATION_NAME` | yes* | Firebolt external location name (must match `CREATE LOCATION`; e.g. `sprinto_s3`) |
|
|
165
|
+
| `S3_BUCKET` | yes | Staging bucket |
|
|
166
|
+
| `S3_PREFIX` | no | Key prefix (default: `dlt-landing`) |
|
|
167
|
+
| `DLT_DATASET_NAME` | no | Demo dataset (default: `demo`) |
|
|
168
|
+
|
|
169
|
+
Credentials belong in `.env` (gitignored) or `.dlt/secrets.toml` (gitignored). See `.dlt/secrets.toml.example`.
|
|
170
|
+
|
|
171
|
+
## Tests
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
pip install -r requirements-dev.txt
|
|
175
|
+
pytest
|
|
176
|
+
|
|
177
|
+
# Optional: live Firebolt + S3 (requires .env and AWS creds)
|
|
178
|
+
FIREBOLT_RUN_INTEGRATION=1 pytest -m integration -v
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
## Roadmap
|
|
182
|
+
|
|
183
|
+
- [x] Package as installable module (`pip install -e .` / `dlt-firebolt`)
|
|
184
|
+
- [x] Config via env vars and `.dlt/secrets.toml` (both live-tested)
|
|
185
|
+
- [x] Merge/append/replace dispositions (merge via delete-insert; replace via truncate-and-insert or insert-from-staging)
|
|
186
|
+
- [x] Unit tests for COPY and merge SQL generation
|
|
187
|
+
- [x] Integration test harness (env-gated)
|
|
188
|
+
- [x] Destination README (dlt-style setup doc)
|
|
189
|
+
- [ ] PyPI publish (`pip install dlt-firebolt` from PyPI)
|
|
190
|
+
- [ ] Upstream PR to [dlt](https://github.com/dlt-hub/dlt) or community listing
|
|
191
|
+
|
|
192
|
+
## Related
|
|
193
|
+
|
|
194
|
+
Customer connector demos that consume this pattern live separately in [sprinto-connectors](https://github.com/firebolt-analytics/sprinto-connectors) (private).
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
# dlt-firebolt
|
|
2
|
+
|
|
3
|
+
Prototype [dlt](https://dlthub.com/) destination for [Firebolt](https://www.firebolt.io/).
|
|
4
|
+
|
|
5
|
+
Loads dlt pipelines into Firebolt using **filesystem staging (Parquet on S3) + `COPY INTO`**, the same pattern as dlt's Snowflake and Redshift destinations.
|
|
6
|
+
|
|
7
|
+
## Status
|
|
8
|
+
|
|
9
|
+
Spike complete. Hardening done; packaging and upstream prep in progress.
|
|
10
|
+
|
|
11
|
+
| Phase | What it proved |
|
|
12
|
+
|-------|----------------|
|
|
13
|
+
| 1 | dlt → S3 Parquet → manual COPY INTO |
|
|
14
|
+
| 2 | Generic `sqlalchemy` destination is not viable on Firebolt |
|
|
15
|
+
| 3 | Native `destination="firebolt"` end-to-end |
|
|
16
|
+
| 4 | Append / merge / replace disposition scripts |
|
|
17
|
+
|
|
18
|
+
See [SPIKE.md](SPIKE.md) for spike notes.
|
|
19
|
+
|
|
20
|
+
## License
|
|
21
|
+
|
|
22
|
+
Apache License 2.0 — see [LICENSE](LICENSE).
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
python -m venv .venv
|
|
28
|
+
source .venv/bin/activate
|
|
29
|
+
pip install -e ".[dev]"
|
|
30
|
+
cp .env.example .env # fill in Firebolt + S3 creds
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Or install dependencies only (no editable package):
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install -r requirements.txt
|
|
37
|
+
pip install -r requirements-dev.txt
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Quick start (Phase 3 demo)
|
|
41
|
+
|
|
42
|
+
Requires:
|
|
43
|
+
|
|
44
|
+
- Firebolt `CREATE LOCATION` for your S3 bucket — set `FIREBOLT_S3_LOCATION_NAME` to the location name (e.g. `sprinto_s3`)
|
|
45
|
+
- HubSpot private app token in `.env` (demo only)
|
|
46
|
+
- AWS credentials for S3 staging
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
export AWS_PROFILE=your-profile
|
|
50
|
+
python phase3_hubspot_to_firebolt.py
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Optional: copy `.dlt/secrets.toml.example` to `.dlt/secrets.toml` and run with `DLT_USE_SECRETS=1`.
|
|
54
|
+
|
|
55
|
+
Before running demos, validate credentials:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
python check_firebolt_env.py
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### Disposition checks (Phase 4)
|
|
62
|
+
|
|
63
|
+
Run each command separately (do not paste inline comments):
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
python phase4_dispositions.py --mode merge
|
|
67
|
+
python phase4_dispositions.py --mode append
|
|
68
|
+
python phase4_dispositions.py --mode append
|
|
69
|
+
python phase4_dispositions.py --mode replace
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
For append, run the command twice and confirm the row count grows.
|
|
73
|
+
|
|
74
|
+
Verify in Firebolt (default dataset `demo`):
|
|
75
|
+
|
|
76
|
+
```sql
|
|
77
|
+
SELECT COUNT(*) FROM demo_hubspot_contacts;
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Usage in a dlt pipeline
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
import sys
|
|
84
|
+
from pathlib import Path
|
|
85
|
+
|
|
86
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
87
|
+
|
|
88
|
+
import dlt
|
|
89
|
+
from firebolt_dest.configuration import make_firebolt_pipeline
|
|
90
|
+
|
|
91
|
+
pipeline = make_firebolt_pipeline(
|
|
92
|
+
pipeline_name="my_pipeline",
|
|
93
|
+
dataset_name="my_dataset",
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
pipeline.run(my_resource(), loader_file_format="parquet")
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Or with `.dlt/secrets.toml`:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
pipeline = make_firebolt_pipeline(
|
|
103
|
+
pipeline_name="my_pipeline",
|
|
104
|
+
dataset_name="my_dataset",
|
|
105
|
+
from_secrets=True,
|
|
106
|
+
)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Tables land as `{dataset}_{table}` (e.g. `my_dataset_orders`).
|
|
110
|
+
|
|
111
|
+
Connection details from environment variables — see [.env.example](.env.example) — or from `.dlt/secrets.toml` — see [.dlt/secrets.toml.example](.dlt/secrets.toml.example).
|
|
112
|
+
|
|
113
|
+
## Layout
|
|
114
|
+
|
|
115
|
+
```
|
|
116
|
+
firebolt_dest/ # destination implementation (fork Redshift COPY pattern)
|
|
117
|
+
factory.py # registers destination="firebolt"
|
|
118
|
+
client.py # COPY load jobs
|
|
119
|
+
sql_client.py # Firebolt SQLAlchemy client
|
|
120
|
+
copy_sql.py # COPY INTO SQL generation
|
|
121
|
+
configuration.py # credentials + S3 location config
|
|
122
|
+
phase1_*.py # spike: dlt → S3 only
|
|
123
|
+
phase2_*.py # spike: dialect smoke test
|
|
124
|
+
phase3_*.py # spike: full native destination demo
|
|
125
|
+
phase4_*.py # append / merge / replace disposition checks
|
|
126
|
+
.dlt/config.toml # non-sensitive dlt defaults (parquet loader)
|
|
127
|
+
.dlt/secrets.toml.example
|
|
128
|
+
tests/ # unit tests (no Firebolt connection)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Configuration
|
|
132
|
+
|
|
133
|
+
| Variable | Required | Description |
|
|
134
|
+
|----------|----------|-------------|
|
|
135
|
+
| `FIREBOLT_CLIENT_ID` | yes | Service account client ID |
|
|
136
|
+
| `FIREBOLT_CLIENT_SECRET` | yes | Service account secret |
|
|
137
|
+
| `FIREBOLT_ACCOUNT_NAME` | yes | Firebolt account name |
|
|
138
|
+
| `FIREBOLT_DATABASE` | yes | Target database |
|
|
139
|
+
| `FIREBOLT_ENGINE` | yes | Engine name |
|
|
140
|
+
| `FIREBOLT_S3_LOCATION_NAME` | yes* | Firebolt external location name (must match `CREATE LOCATION`; e.g. `sprinto_s3`) |
|
|
141
|
+
| `S3_BUCKET` | yes | Staging bucket |
|
|
142
|
+
| `S3_PREFIX` | no | Key prefix (default: `dlt-landing`) |
|
|
143
|
+
| `DLT_DATASET_NAME` | no | Demo dataset (default: `demo`) |
|
|
144
|
+
|
|
145
|
+
Credentials belong in `.env` (gitignored) or `.dlt/secrets.toml` (gitignored). See `.dlt/secrets.toml.example`.
|
|
146
|
+
|
|
147
|
+
## Tests
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
pip install -r requirements-dev.txt
|
|
151
|
+
pytest
|
|
152
|
+
|
|
153
|
+
# Optional: live Firebolt + S3 (requires .env and AWS creds)
|
|
154
|
+
FIREBOLT_RUN_INTEGRATION=1 pytest -m integration -v
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Roadmap
|
|
158
|
+
|
|
159
|
+
- [x] Package as installable module (`pip install -e .` / `dlt-firebolt`)
|
|
160
|
+
- [x] Config via env vars and `.dlt/secrets.toml` (both live-tested)
|
|
161
|
+
- [x] Merge/append/replace dispositions (merge via delete-insert; replace via truncate-and-insert or insert-from-staging)
|
|
162
|
+
- [x] Unit tests for COPY and merge SQL generation
|
|
163
|
+
- [x] Integration test harness (env-gated)
|
|
164
|
+
- [x] Destination README (dlt-style setup doc)
|
|
165
|
+
- [ ] PyPI publish (`pip install dlt-firebolt` from PyPI)
|
|
166
|
+
- [ ] Upstream PR to [dlt](https://github.com/dlt-hub/dlt) or community listing
|
|
167
|
+
|
|
168
|
+
## Related
|
|
169
|
+
|
|
170
|
+
Customer connector demos that consume this pattern live separately in [sprinto-connectors](https://github.com/firebolt-analytics/sprinto-connectors) (private).
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# dlt × Firebolt spike
|
|
2
|
+
|
|
3
|
+
Benjamin asked: how hard is a native Firebolt destination?
|
|
4
|
+
|
|
5
|
+
**Goal:** prove dlt replaces connector boilerplate, then scope a upstreamable `destination="firebolt"`.
|
|
6
|
+
|
|
7
|
+
## Phase 1 — dlt → S3
|
|
8
|
+
|
|
9
|
+
Prove dlt handles schema, state, incrementals. You run COPY INTO separately.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
cd ~/dlt-firebolt-spike
|
|
13
|
+
python -m venv .venv && source .venv/bin/activate
|
|
14
|
+
pip install -r requirements.txt
|
|
15
|
+
cp .env.example .env # fill in creds
|
|
16
|
+
export AWS_PROFILE=your-profile
|
|
17
|
+
python phase1_hubspot_to_s3.py
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
**Success:** Parquet lands in S3; row counts match source after COPY.
|
|
21
|
+
|
|
22
|
+
## Phase 2 — Dialect smoke test
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
python phase2_dialect_smoke.py
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
**Expected:** literal + parametrized INSERT pass; dlt sqlalchemy load fails on schema sync.
|
|
29
|
+
**Implication:** build native destination with CREATE TABLE + staged COPY (Phase 3).
|
|
30
|
+
|
|
31
|
+
## Phase 3 — Native destination prototype
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
export AWS_PROFILE=your-profile
|
|
35
|
+
python phase3_hubspot_to_firebolt.py
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Verify: `SELECT COUNT(*) FROM demo_hubspot_contacts;` (or your `DLT_DATASET_NAME`).
|
|
39
|
+
|
|
40
|
+
**Module:** `firebolt_dest/` — pip package `dlt-firebolt`; see `firebolt_dest/README.md`.
|
|
41
|
+
|
|
42
|
+
## Upstream checklist (Benjamin)
|
|
43
|
+
|
|
44
|
+
| Item | Status |
|
|
45
|
+
|------|--------|
|
|
46
|
+
| E2E prototype (`destination="firebolt"`) | Done |
|
|
47
|
+
| Merge / append / replace dispositions | Done (live validated) |
|
|
48
|
+
| Config (env vars + secrets.toml) | Done (env live-tested; secrets.toml live-tested after host/database fix) |
|
|
49
|
+
| dlt internal tables (loads, state, version) | Done (state sync works) |
|
|
50
|
+
| Unit tests | Done |
|
|
51
|
+
| pip installable package | Done |
|
|
52
|
+
| Integration test harness | Done (opt-in via env) |
|
|
53
|
+
| Destination setup docs | Done |
|
|
54
|
+
| PyPI publish | Pending |
|
|
55
|
+
| PR to dlt-hub/dlt | Pending (dlt policy: new destinations rarely merged; community package path) |
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Validate Firebolt + S3 env before running phase3/phase4."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import sqlalchemy as sa
|
|
11
|
+
from dotenv import load_dotenv
|
|
12
|
+
|
|
13
|
+
ROOT = Path(__file__).resolve().parent
|
|
14
|
+
sys.path.insert(0, str(ROOT))
|
|
15
|
+
|
|
16
|
+
from firebolt_dest.configuration import (
|
|
17
|
+
firebolt_url_from_env,
|
|
18
|
+
s3_location_name_from_env,
|
|
19
|
+
staging_bucket_url,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
load_dotenv(ROOT / ".env")
|
|
23
|
+
|
|
24
|
+
REQUIRED = (
|
|
25
|
+
"FIREBOLT_CLIENT_ID",
|
|
26
|
+
"FIREBOLT_CLIENT_SECRET",
|
|
27
|
+
"FIREBOLT_ACCOUNT_NAME",
|
|
28
|
+
"FIREBOLT_DATABASE",
|
|
29
|
+
"FIREBOLT_ENGINE",
|
|
30
|
+
"S3_BUCKET",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _mask(value: str) -> str:
|
|
35
|
+
value = value.strip()
|
|
36
|
+
if len(value) <= 4:
|
|
37
|
+
return "****"
|
|
38
|
+
return f"{value[:2]}...{value[-2:]}"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def main() -> int:
|
|
42
|
+
print("Firebolt / S3 environment check\n")
|
|
43
|
+
|
|
44
|
+
missing = [name for name in REQUIRED if not os.environ.get(name, "").strip()]
|
|
45
|
+
if missing:
|
|
46
|
+
print("Missing required variables:")
|
|
47
|
+
for name in missing:
|
|
48
|
+
print(f" - {name}")
|
|
49
|
+
return 1
|
|
50
|
+
|
|
51
|
+
for name in REQUIRED:
|
|
52
|
+
print(f" {name} = {_mask(os.environ[name])}")
|
|
53
|
+
|
|
54
|
+
print(f" FIREBOLT_S3_LOCATION_NAME = {s3_location_name_from_env()}")
|
|
55
|
+
print(f" S3_PREFIX = {os.environ.get('S3_PREFIX', 'dlt-landing')}")
|
|
56
|
+
print(f" staging bucket_url = {staging_bucket_url()}")
|
|
57
|
+
print()
|
|
58
|
+
|
|
59
|
+
url = firebolt_url_from_env()
|
|
60
|
+
print("Testing Firebolt OAuth + SQL connection...")
|
|
61
|
+
try:
|
|
62
|
+
engine = sa.create_engine(url)
|
|
63
|
+
with engine.connect() as conn:
|
|
64
|
+
conn.execute(sa.text("SELECT 1"))
|
|
65
|
+
print(" OK: Firebolt credentials accepted.")
|
|
66
|
+
except Exception as exc:
|
|
67
|
+
print(" FAIL: Firebolt authentication or connection failed.")
|
|
68
|
+
print(f" {type(exc).__name__}: {exc}")
|
|
69
|
+
print()
|
|
70
|
+
print("Fix:")
|
|
71
|
+
print(" 1. Firebolt console → Settings → Service accounts")
|
|
72
|
+
print(" 2. Create a new service account or rotate the secret")
|
|
73
|
+
print(" 3. Update FIREBOLT_CLIENT_ID and FIREBOLT_CLIENT_SECRET in .env")
|
|
74
|
+
print(" 4. Confirm FIREBOLT_ACCOUNT_NAME, FIREBOLT_DATABASE, FIREBOLT_ENGINE")
|
|
75
|
+
return 2
|
|
76
|
+
|
|
77
|
+
location_name = s3_location_name_from_env()
|
|
78
|
+
print(f"Testing Firebolt external location '{location_name}'...")
|
|
79
|
+
try:
|
|
80
|
+
engine = sa.create_engine(url)
|
|
81
|
+
with engine.connect() as conn:
|
|
82
|
+
rows = conn.execute(
|
|
83
|
+
sa.text(
|
|
84
|
+
"SELECT 1 FROM information_schema.locations "
|
|
85
|
+
"WHERE location_name = :name"
|
|
86
|
+
),
|
|
87
|
+
{"name": location_name},
|
|
88
|
+
).fetchall()
|
|
89
|
+
if not rows:
|
|
90
|
+
print(f" FAIL: Location '{location_name}' not found in this database.")
|
|
91
|
+
print()
|
|
92
|
+
print("Fix:")
|
|
93
|
+
print(" 1. Run CREATE LOCATION in Firebolt for your S3 bucket")
|
|
94
|
+
print(" 2. Set FIREBOLT_S3_LOCATION_NAME to that exact location name")
|
|
95
|
+
return 4
|
|
96
|
+
print(f" OK: Location '{location_name}' exists.")
|
|
97
|
+
except Exception as exc:
|
|
98
|
+
print(f" WARN: Could not verify location ({type(exc).__name__}: {exc})")
|
|
99
|
+
|
|
100
|
+
print()
|
|
101
|
+
print("Testing S3 staging access...")
|
|
102
|
+
try:
|
|
103
|
+
import fsspec
|
|
104
|
+
|
|
105
|
+
fs, _ = fsspec.core.url_to_fs(staging_bucket_url())
|
|
106
|
+
fs.ls(fs._strip_protocol(staging_bucket_url()), detail=False)
|
|
107
|
+
print(" OK: S3 staging bucket reachable.")
|
|
108
|
+
except Exception as exc:
|
|
109
|
+
print(" FAIL: S3 staging not reachable with current AWS credentials.")
|
|
110
|
+
print(f" {type(exc).__name__}: {exc}")
|
|
111
|
+
print()
|
|
112
|
+
print("Fix:")
|
|
113
|
+
print(" export AWS_PROFILE=sandbox")
|
|
114
|
+
print(" aws sso login --profile sandbox")
|
|
115
|
+
return 3
|
|
116
|
+
|
|
117
|
+
print()
|
|
118
|
+
print("All checks passed. You can run phase3/phase4.")
|
|
119
|
+
return 0
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
if __name__ == "__main__":
|
|
123
|
+
raise SystemExit(main())
|