dlt-firebolt 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. dlt_firebolt-0.1.0/.dlt/config.toml +3 -0
  2. dlt_firebolt-0.1.0/.dlt/secrets.toml.example +20 -0
  3. dlt_firebolt-0.1.0/.env.example +17 -0
  4. dlt_firebolt-0.1.0/.github/workflows/ci.yml +27 -0
  5. dlt_firebolt-0.1.0/.github/workflows/publish.yml +39 -0
  6. dlt_firebolt-0.1.0/.gitignore +12 -0
  7. dlt_firebolt-0.1.0/LICENSE +18 -0
  8. dlt_firebolt-0.1.0/PKG-INFO +194 -0
  9. dlt_firebolt-0.1.0/README.md +170 -0
  10. dlt_firebolt-0.1.0/SPIKE.md +55 -0
  11. dlt_firebolt-0.1.0/check_firebolt_env.py +123 -0
  12. dlt_firebolt-0.1.0/firebolt_dest/README.md +102 -0
  13. dlt_firebolt-0.1.0/firebolt_dest/__init__.py +13 -0
  14. dlt_firebolt-0.1.0/firebolt_dest/client.py +154 -0
  15. dlt_firebolt-0.1.0/firebolt_dest/configuration.py +125 -0
  16. dlt_firebolt-0.1.0/firebolt_dest/copy_sql.py +29 -0
  17. dlt_firebolt-0.1.0/firebolt_dest/factory.py +115 -0
  18. dlt_firebolt-0.1.0/firebolt_dest/sql_client.py +181 -0
  19. dlt_firebolt-0.1.0/phase1_copy.sql +11 -0
  20. dlt_firebolt-0.1.0/phase1_hubspot_to_s3.py +173 -0
  21. dlt_firebolt-0.1.0/phase2_dialect_smoke.py +149 -0
  22. dlt_firebolt-0.1.0/phase3_hubspot_to_firebolt.py +100 -0
  23. dlt_firebolt-0.1.0/phase4_dispositions.py +74 -0
  24. dlt_firebolt-0.1.0/phase5_nested.py +71 -0
  25. dlt_firebolt-0.1.0/pyproject.toml +43 -0
  26. dlt_firebolt-0.1.0/requirements-dev.txt +1 -0
  27. dlt_firebolt-0.1.0/requirements.txt +4 -0
  28. dlt_firebolt-0.1.0/tests/conftest.py +13 -0
  29. dlt_firebolt-0.1.0/tests/test_capabilities.py +11 -0
  30. dlt_firebolt-0.1.0/tests/test_copy_sql.py +24 -0
  31. dlt_firebolt-0.1.0/tests/test_credentials.py +19 -0
  32. dlt_firebolt-0.1.0/tests/test_integration.py +45 -0
  33. dlt_firebolt-0.1.0/tests/test_merge_sql.py +114 -0
  34. dlt_firebolt-0.1.0/tests/test_sql_client.py +57 -0
@@ -0,0 +1,3 @@
1
+ # Non-sensitive dlt defaults for this destination prototype.
2
+ [pipeline]
3
+ loader_file_format = "parquet"
@@ -0,0 +1,20 @@
1
+ # Copy to .dlt/secrets.toml (gitignored). Do not commit real credentials.
2
+
3
+ [destination.firebolt]
4
+ # Option A: connection string (recommended)
5
+ # credentials = "firebolt://CLIENT_ID:CLIENT_SECRET@DATABASE/ENGINE?account_name=ACCOUNT"
6
+
7
+ s3_location_name = "your_location_name"
8
+ s3_prefix = "dlt-landing"
9
+
10
+ # Option B: structured credentials
11
+ # Firebolt maps to SQLAlchemy URL as host=DATABASE, database=ENGINE (see firebolt_url_from_env).
12
+ [destination.firebolt.credentials]
13
+ host = "YOUR_DATABASE"
14
+ database = "YOUR_ENGINE"
15
+ username = "YOUR_CLIENT_ID"
16
+ password = "YOUR_CLIENT_SECRET"
17
+ account_name = "YOUR_ACCOUNT_NAME"
18
+
19
+ [destination.filesystem]
20
+ bucket_url = "s3://YOUR_BUCKET/dlt-landing/dlt/staging"
@@ -0,0 +1,17 @@
1
+ # Firebolt service account (Settings → Service accounts)
2
+ FIREBOLT_CLIENT_ID=
3
+ FIREBOLT_CLIENT_SECRET=
4
+ FIREBOLT_ACCOUNT_NAME=
5
+ FIREBOLT_DATABASE=
6
+ FIREBOLT_ENGINE=
7
+
8
+ # S3 staging — must match the name from your Firebolt CREATE LOCATION statement
9
+ # (e.g. CREATE LOCATION "sprinto_s3" WITH SOURCE='CLOUD_STORAGE' URL='s3://...')
10
+ FIREBOLT_S3_LOCATION_NAME=sprinto_s3
11
+ S3_BUCKET=
12
+ S3_PREFIX=dlt-landing
13
+
14
+ # Optional: HubSpot demo scripts only (phase1 / phase3)
15
+ HUBSPOT_ACCESS_TOKEN=
16
+
17
+ # AWS credentials for S3 staging (e.g. export AWS_PROFILE=your-profile)
@@ -0,0 +1,27 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ matrix:
13
+ python-version: ["3.10", "3.12"]
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+
17
+ - uses: actions/setup-python@v5
18
+ with:
19
+ python-version: ${{ matrix.python-version }}
20
+
21
+ - name: Install package and test deps
22
+ run: |
23
+ python -m pip install --upgrade pip
24
+ pip install -e ".[dev]"
25
+
26
+ - name: Unit tests
27
+ run: pytest -q -m "not integration"
@@ -0,0 +1,39 @@
1
+ name: Publish to PyPI
2
+
3
+ # Mirrors firebolt-db Python SDK auth: TWINE_* from PYPI_USERNAME / PYPI_PASSWORD.
4
+ # Prerequisite: add those secrets to firebolt-analytics/dlt-firebolt (org secrets
5
+ # do not cross from firebolt-db). Bump version in pyproject.toml before release.
6
+
7
+ on:
8
+ release:
9
+ types: [published]
10
+ workflow_dispatch:
11
+
12
+ jobs:
13
+ publish:
14
+ runs-on: ubuntu-latest
15
+ permissions:
16
+ contents: read
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+
20
+ - uses: actions/setup-python@v5
21
+ with:
22
+ python-version: "3.12"
23
+
24
+ - name: Install build tooling
25
+ run: |
26
+ python -m pip install --upgrade pip
27
+ pip install -e ".[dev]" build twine
28
+
29
+ - name: Unit tests
30
+ run: pytest -q -m "not integration"
31
+
32
+ - name: Build
33
+ run: python -m build
34
+
35
+ - name: Publish
36
+ env:
37
+ TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
38
+ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
39
+ run: twine upload dist/*
@@ -0,0 +1,12 @@
1
+ .venv/
2
+ .env
3
+ .dlt/secrets.toml
4
+ .dlt/pipelines/
5
+ __pycache__/
6
+ *.py[cod]
7
+ *.egg-info/
8
+ .pytest_cache/
9
+ .mypy_cache/
10
+ build/
11
+ dist/
12
+ .DS_Store
@@ -0,0 +1,18 @@
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ Copyright 2026 Firebolt Analytics Inc.
7
+
8
+ Licensed under the Apache License, Version 2.0 (the "License");
9
+ you may not use this file except in compliance with the License.
10
+ You may obtain a copy of the License at
11
+
12
+ http://www.apache.org/licenses/LICENSE-2.0
13
+
14
+ Unless required by applicable law or agreed to in writing, software
15
+ distributed under the License is distributed on an "AS IS" BASIS,
16
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ See the License for the specific language governing permissions and
18
+ limitations under the License.
@@ -0,0 +1,194 @@
1
+ Metadata-Version: 2.4
2
+ Name: dlt-firebolt
3
+ Version: 0.1.0
4
+ Summary: dlt destination for Firebolt (staged Parquet + COPY INTO)
5
+ Project-URL: Homepage, https://github.com/firebolt-analytics/dlt-firebolt
6
+ Project-URL: Repository, https://github.com/firebolt-analytics/dlt-firebolt
7
+ Author: Firebolt Analytics
8
+ License: Apache-2.0
9
+ License-File: LICENSE
10
+ Keywords: data-pipeline,dlt,etl,firebolt
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: Apache Software License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Topic :: Database
16
+ Requires-Python: >=3.10
17
+ Requires-Dist: dlt[parquet,s3,sqlalchemy]>=1.0.0
18
+ Requires-Dist: firebolt-sqlalchemy
19
+ Provides-Extra: dev
20
+ Requires-Dist: pytest>=8.0; extra == 'dev'
21
+ Requires-Dist: python-dotenv; extra == 'dev'
22
+ Requires-Dist: requests; extra == 'dev'
23
+ Description-Content-Type: text/markdown
24
+
25
+ # dlt-firebolt
26
+
27
+ Prototype [dlt](https://dlthub.com/) destination for [Firebolt](https://www.firebolt.io/).
28
+
29
+ Loads dlt pipelines into Firebolt using **filesystem staging (Parquet on S3) + `COPY INTO`**, the same pattern as dlt's Snowflake and Redshift destinations.
30
+
31
+ ## Status
32
+
33
+ Spike complete. Hardening done; packaging and upstream prep in progress.
34
+
35
+ | Phase | What it proved |
36
+ |-------|----------------|
37
+ | 1 | dlt → S3 Parquet → manual COPY INTO |
38
+ | 2 | Generic `sqlalchemy` destination is not viable on Firebolt |
39
+ | 3 | Native `destination="firebolt"` end-to-end |
40
+ | 4 | Append / merge / replace disposition scripts |
41
+
42
+ See [SPIKE.md](SPIKE.md) for spike notes.
43
+
44
+ ## License
45
+
46
+ Apache License 2.0 — see [LICENSE](LICENSE).
47
+
48
+ ## Install
49
+
50
+ ```bash
51
+ python -m venv .venv
52
+ source .venv/bin/activate
53
+ pip install -e ".[dev]"
54
+ cp .env.example .env # fill in Firebolt + S3 creds
55
+ ```
56
+
57
+ Or install dependencies only (no editable package):
58
+
59
+ ```bash
60
+ pip install -r requirements.txt
61
+ pip install -r requirements-dev.txt
62
+ ```
63
+
64
+ ## Quick start (Phase 3 demo)
65
+
66
+ Requires:
67
+
68
+ - Firebolt `CREATE LOCATION` for your S3 bucket — set `FIREBOLT_S3_LOCATION_NAME` to the location name (e.g. `sprinto_s3`)
69
+ - HubSpot private app token in `.env` (demo only)
70
+ - AWS credentials for S3 staging
71
+
72
+ ```bash
73
+ export AWS_PROFILE=your-profile
74
+ python phase3_hubspot_to_firebolt.py
75
+ ```
76
+
77
+ Optional: copy `.dlt/secrets.toml.example` to `.dlt/secrets.toml` and run with `DLT_USE_SECRETS=1`.
78
+
79
+ Before running demos, validate credentials:
80
+
81
+ ```bash
82
+ python check_firebolt_env.py
83
+ ```
84
+
85
+ ### Disposition checks (Phase 4)
86
+
87
+ Run each command separately (do not paste inline comments):
88
+
89
+ ```bash
90
+ python phase4_dispositions.py --mode merge
91
+ python phase4_dispositions.py --mode append
92
+ python phase4_dispositions.py --mode append
93
+ python phase4_dispositions.py --mode replace
94
+ ```
95
+
96
+ For append, run the command twice and confirm the row count grows.
97
+
98
+ Verify in Firebolt (default dataset `demo`):
99
+
100
+ ```sql
101
+ SELECT COUNT(*) FROM demo_hubspot_contacts;
102
+ ```
103
+
104
+ ## Usage in a dlt pipeline
105
+
106
+ ```python
107
+ import sys
108
+ from pathlib import Path
109
+
110
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
111
+
112
+ import dlt
113
+ from firebolt_dest.configuration import make_firebolt_pipeline
114
+
115
+ pipeline = make_firebolt_pipeline(
116
+ pipeline_name="my_pipeline",
117
+ dataset_name="my_dataset",
118
+ )
119
+
120
+ pipeline.run(my_resource(), loader_file_format="parquet")
121
+ ```
122
+
123
+ Or with `.dlt/secrets.toml`:
124
+
125
+ ```python
126
+ pipeline = make_firebolt_pipeline(
127
+ pipeline_name="my_pipeline",
128
+ dataset_name="my_dataset",
129
+ from_secrets=True,
130
+ )
131
+ ```
132
+
133
+ Tables land as `{dataset}_{table}` (e.g. `my_dataset_orders`).
134
+
135
+ Connection details from environment variables — see [.env.example](.env.example) — or from `.dlt/secrets.toml` — see [.dlt/secrets.toml.example](.dlt/secrets.toml.example).
136
+
137
+ ## Layout
138
+
139
+ ```
140
+ firebolt_dest/ # destination implementation (fork Redshift COPY pattern)
141
+ factory.py # registers destination="firebolt"
142
+ client.py # COPY load jobs
143
+ sql_client.py # Firebolt SQLAlchemy client
144
+ copy_sql.py # COPY INTO SQL generation
145
+ configuration.py # credentials + S3 location config
146
+ phase1_*.py # spike: dlt → S3 only
147
+ phase2_*.py # spike: dialect smoke test
148
+ phase3_*.py # spike: full native destination demo
149
+ phase4_*.py # append / merge / replace disposition checks
150
+ .dlt/config.toml # non-sensitive dlt defaults (parquet loader)
151
+ .dlt/secrets.toml.example
152
+ tests/ # unit tests (no Firebolt connection)
153
+ ```
154
+
155
+ ## Configuration
156
+
157
+ | Variable | Required | Description |
158
+ |----------|----------|-------------|
159
+ | `FIREBOLT_CLIENT_ID` | yes | Service account client ID |
160
+ | `FIREBOLT_CLIENT_SECRET` | yes | Service account secret |
161
+ | `FIREBOLT_ACCOUNT_NAME` | yes | Firebolt account name |
162
+ | `FIREBOLT_DATABASE` | yes | Target database |
163
+ | `FIREBOLT_ENGINE` | yes | Engine name |
164
+ | `FIREBOLT_S3_LOCATION_NAME` | yes* | Firebolt external location name (must match `CREATE LOCATION`; e.g. `sprinto_s3`) |
165
+ | `S3_BUCKET` | yes | Staging bucket |
166
+ | `S3_PREFIX` | no | Key prefix (default: `dlt-landing`) |
167
+ | `DLT_DATASET_NAME` | no | Demo dataset (default: `demo`) |
168
+
169
+ Credentials belong in `.env` (gitignored) or `.dlt/secrets.toml` (gitignored). See `.dlt/secrets.toml.example`.
170
+
171
+ ## Tests
172
+
173
+ ```bash
174
+ pip install -r requirements-dev.txt
175
+ pytest
176
+
177
+ # Optional: live Firebolt + S3 (requires .env and AWS creds)
178
+ FIREBOLT_RUN_INTEGRATION=1 pytest -m integration -v
179
+ ```
180
+
181
+ ## Roadmap
182
+
183
+ - [x] Package as installable module (`pip install -e .` / `dlt-firebolt`)
184
+ - [x] Config via env vars and `.dlt/secrets.toml` (both live-tested)
185
+ - [x] Merge/append/replace dispositions (merge via delete-insert; replace via truncate-and-insert or insert-from-staging)
186
+ - [x] Unit tests for COPY and merge SQL generation
187
+ - [x] Integration test harness (env-gated)
188
+ - [x] Destination README (dlt-style setup doc)
189
+ - [ ] PyPI publish (`pip install dlt-firebolt` from PyPI)
190
+ - [ ] Upstream PR to [dlt](https://github.com/dlt-hub/dlt) or community listing
191
+
192
+ ## Related
193
+
194
+ Customer connector demos that consume this pattern live separately in [sprinto-connectors](https://github.com/firebolt-analytics/sprinto-connectors) (private).
@@ -0,0 +1,170 @@
1
+ # dlt-firebolt
2
+
3
+ Prototype [dlt](https://dlthub.com/) destination for [Firebolt](https://www.firebolt.io/).
4
+
5
+ Loads dlt pipelines into Firebolt using **filesystem staging (Parquet on S3) + `COPY INTO`**, the same pattern as dlt's Snowflake and Redshift destinations.
6
+
7
+ ## Status
8
+
9
+ Spike complete. Hardening done; packaging and upstream prep in progress.
10
+
11
+ | Phase | What it proved |
12
+ |-------|----------------|
13
+ | 1 | dlt → S3 Parquet → manual COPY INTO |
14
+ | 2 | Generic `sqlalchemy` destination is not viable on Firebolt |
15
+ | 3 | Native `destination="firebolt"` end-to-end |
16
+ | 4 | Append / merge / replace disposition scripts |
17
+
18
+ See [SPIKE.md](SPIKE.md) for spike notes.
19
+
20
+ ## License
21
+
22
+ Apache License 2.0 — see [LICENSE](LICENSE).
23
+
24
+ ## Install
25
+
26
+ ```bash
27
+ python -m venv .venv
28
+ source .venv/bin/activate
29
+ pip install -e ".[dev]"
30
+ cp .env.example .env # fill in Firebolt + S3 creds
31
+ ```
32
+
33
+ Or install dependencies only (no editable package):
34
+
35
+ ```bash
36
+ pip install -r requirements.txt
37
+ pip install -r requirements-dev.txt
38
+ ```
39
+
40
+ ## Quick start (Phase 3 demo)
41
+
42
+ Requires:
43
+
44
+ - Firebolt `CREATE LOCATION` for your S3 bucket — set `FIREBOLT_S3_LOCATION_NAME` to the location name (e.g. `sprinto_s3`)
45
+ - HubSpot private app token in `.env` (demo only)
46
+ - AWS credentials for S3 staging
47
+
48
+ ```bash
49
+ export AWS_PROFILE=your-profile
50
+ python phase3_hubspot_to_firebolt.py
51
+ ```
52
+
53
+ Optional: copy `.dlt/secrets.toml.example` to `.dlt/secrets.toml` and run with `DLT_USE_SECRETS=1`.
54
+
55
+ Before running demos, validate credentials:
56
+
57
+ ```bash
58
+ python check_firebolt_env.py
59
+ ```
60
+
61
+ ### Disposition checks (Phase 4)
62
+
63
+ Run each command separately (do not paste inline comments):
64
+
65
+ ```bash
66
+ python phase4_dispositions.py --mode merge
67
+ python phase4_dispositions.py --mode append
68
+ python phase4_dispositions.py --mode append
69
+ python phase4_dispositions.py --mode replace
70
+ ```
71
+
72
+ For append, run the command twice and confirm the row count grows.
73
+
74
+ Verify in Firebolt (default dataset `demo`):
75
+
76
+ ```sql
77
+ SELECT COUNT(*) FROM demo_hubspot_contacts;
78
+ ```
79
+
80
+ ## Usage in a dlt pipeline
81
+
82
+ ```python
83
+ import sys
84
+ from pathlib import Path
85
+
86
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
87
+
88
+ import dlt
89
+ from firebolt_dest.configuration import make_firebolt_pipeline
90
+
91
+ pipeline = make_firebolt_pipeline(
92
+ pipeline_name="my_pipeline",
93
+ dataset_name="my_dataset",
94
+ )
95
+
96
+ pipeline.run(my_resource(), loader_file_format="parquet")
97
+ ```
98
+
99
+ Or with `.dlt/secrets.toml`:
100
+
101
+ ```python
102
+ pipeline = make_firebolt_pipeline(
103
+ pipeline_name="my_pipeline",
104
+ dataset_name="my_dataset",
105
+ from_secrets=True,
106
+ )
107
+ ```
108
+
109
+ Tables land as `{dataset}_{table}` (e.g. `my_dataset_orders`).
110
+
111
+ Connection details from environment variables — see [.env.example](.env.example) — or from `.dlt/secrets.toml` — see [.dlt/secrets.toml.example](.dlt/secrets.toml.example).
112
+
113
+ ## Layout
114
+
115
+ ```
116
+ firebolt_dest/ # destination implementation (fork Redshift COPY pattern)
117
+ factory.py # registers destination="firebolt"
118
+ client.py # COPY load jobs
119
+ sql_client.py # Firebolt SQLAlchemy client
120
+ copy_sql.py # COPY INTO SQL generation
121
+ configuration.py # credentials + S3 location config
122
+ phase1_*.py # spike: dlt → S3 only
123
+ phase2_*.py # spike: dialect smoke test
124
+ phase3_*.py # spike: full native destination demo
125
+ phase4_*.py # append / merge / replace disposition checks
126
+ .dlt/config.toml # non-sensitive dlt defaults (parquet loader)
127
+ .dlt/secrets.toml.example
128
+ tests/ # unit tests (no Firebolt connection)
129
+ ```
130
+
131
+ ## Configuration
132
+
133
+ | Variable | Required | Description |
134
+ |----------|----------|-------------|
135
+ | `FIREBOLT_CLIENT_ID` | yes | Service account client ID |
136
+ | `FIREBOLT_CLIENT_SECRET` | yes | Service account secret |
137
+ | `FIREBOLT_ACCOUNT_NAME` | yes | Firebolt account name |
138
+ | `FIREBOLT_DATABASE` | yes | Target database |
139
+ | `FIREBOLT_ENGINE` | yes | Engine name |
140
+ | `FIREBOLT_S3_LOCATION_NAME` | yes* | Firebolt external location name (must match `CREATE LOCATION`; e.g. `sprinto_s3`) |
141
+ | `S3_BUCKET` | yes | Staging bucket |
142
+ | `S3_PREFIX` | no | Key prefix (default: `dlt-landing`) |
143
+ | `DLT_DATASET_NAME` | no | Demo dataset (default: `demo`) |
144
+
145
+ Credentials belong in `.env` (gitignored) or `.dlt/secrets.toml` (gitignored). See `.dlt/secrets.toml.example`.
146
+
147
+ ## Tests
148
+
149
+ ```bash
150
+ pip install -r requirements-dev.txt
151
+ pytest
152
+
153
+ # Optional: live Firebolt + S3 (requires .env and AWS creds)
154
+ FIREBOLT_RUN_INTEGRATION=1 pytest -m integration -v
155
+ ```
156
+
157
+ ## Roadmap
158
+
159
+ - [x] Package as installable module (`pip install -e .` / `dlt-firebolt`)
160
+ - [x] Config via env vars and `.dlt/secrets.toml` (both live-tested)
161
+ - [x] Merge/append/replace dispositions (merge via delete-insert; replace via truncate-and-insert or insert-from-staging)
162
+ - [x] Unit tests for COPY and merge SQL generation
163
+ - [x] Integration test harness (env-gated)
164
+ - [x] Destination README (dlt-style setup doc)
165
+ - [ ] PyPI publish (`pip install dlt-firebolt` from PyPI)
166
+ - [ ] Upstream PR to [dlt](https://github.com/dlt-hub/dlt) or community listing
167
+
168
+ ## Related
169
+
170
+ Customer connector demos that consume this pattern live separately in [sprinto-connectors](https://github.com/firebolt-analytics/sprinto-connectors) (private).
@@ -0,0 +1,55 @@
1
+ # dlt × Firebolt spike
2
+
3
+ Benjamin asked: how hard is a native Firebolt destination?
4
+
5
+ **Goal:** prove dlt replaces connector boilerplate, then scope a upstreamable `destination="firebolt"`.
6
+
7
+ ## Phase 1 — dlt → S3
8
+
9
+ Prove dlt handles schema, state, incrementals. You run COPY INTO separately.
10
+
11
+ ```bash
12
+ cd ~/dlt-firebolt-spike
13
+ python -m venv .venv && source .venv/bin/activate
14
+ pip install -r requirements.txt
15
+ cp .env.example .env # fill in creds
16
+ export AWS_PROFILE=your-profile
17
+ python phase1_hubspot_to_s3.py
18
+ ```
19
+
20
+ **Success:** Parquet lands in S3; row counts match source after COPY.
21
+
22
+ ## Phase 2 — Dialect smoke test
23
+
24
+ ```bash
25
+ python phase2_dialect_smoke.py
26
+ ```
27
+
28
+ **Expected:** literal + parametrized INSERT pass; dlt sqlalchemy load fails on schema sync.
29
+ **Implication:** build native destination with CREATE TABLE + staged COPY (Phase 3).
30
+
31
+ ## Phase 3 — Native destination prototype
32
+
33
+ ```bash
34
+ export AWS_PROFILE=your-profile
35
+ python phase3_hubspot_to_firebolt.py
36
+ ```
37
+
38
+ Verify: `SELECT COUNT(*) FROM demo_hubspot_contacts;` (or your `DLT_DATASET_NAME`).
39
+
40
+ **Module:** `firebolt_dest/` — pip package `dlt-firebolt`; see `firebolt_dest/README.md`.
41
+
42
+ ## Upstream checklist (Benjamin)
43
+
44
+ | Item | Status |
45
+ |------|--------|
46
+ | E2E prototype (`destination="firebolt"`) | Done |
47
+ | Merge / append / replace dispositions | Done (live validated) |
48
+ | Config (env vars + secrets.toml) | Done (env live-tested; secrets.toml live-tested after host/database fix) |
49
+ | dlt internal tables (loads, state, version) | Done (state sync works) |
50
+ | Unit tests | Done |
51
+ | pip installable package | Done |
52
+ | Integration test harness | Done (opt-in via env) |
53
+ | Destination setup docs | Done |
54
+ | PyPI publish | Pending |
55
+ | PR to dlt-hub/dlt | Pending (dlt policy: new destinations rarely merged; community package path) |
@@ -0,0 +1,123 @@
1
+ #!/usr/bin/env python3
2
+ """Validate Firebolt + S3 env before running phase3/phase4."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import os
7
+ import sys
8
+ from pathlib import Path
9
+
10
+ import sqlalchemy as sa
11
+ from dotenv import load_dotenv
12
+
13
+ ROOT = Path(__file__).resolve().parent
14
+ sys.path.insert(0, str(ROOT))
15
+
16
+ from firebolt_dest.configuration import (
17
+ firebolt_url_from_env,
18
+ s3_location_name_from_env,
19
+ staging_bucket_url,
20
+ )
21
+
22
+ load_dotenv(ROOT / ".env")
23
+
24
+ REQUIRED = (
25
+ "FIREBOLT_CLIENT_ID",
26
+ "FIREBOLT_CLIENT_SECRET",
27
+ "FIREBOLT_ACCOUNT_NAME",
28
+ "FIREBOLT_DATABASE",
29
+ "FIREBOLT_ENGINE",
30
+ "S3_BUCKET",
31
+ )
32
+
33
+
34
+ def _mask(value: str) -> str:
35
+ value = value.strip()
36
+ if len(value) <= 4:
37
+ return "****"
38
+ return f"{value[:2]}...{value[-2:]}"
39
+
40
+
41
+ def main() -> int:
42
+ print("Firebolt / S3 environment check\n")
43
+
44
+ missing = [name for name in REQUIRED if not os.environ.get(name, "").strip()]
45
+ if missing:
46
+ print("Missing required variables:")
47
+ for name in missing:
48
+ print(f" - {name}")
49
+ return 1
50
+
51
+ for name in REQUIRED:
52
+ print(f" {name} = {_mask(os.environ[name])}")
53
+
54
+ print(f" FIREBOLT_S3_LOCATION_NAME = {s3_location_name_from_env()}")
55
+ print(f" S3_PREFIX = {os.environ.get('S3_PREFIX', 'dlt-landing')}")
56
+ print(f" staging bucket_url = {staging_bucket_url()}")
57
+ print()
58
+
59
+ url = firebolt_url_from_env()
60
+ print("Testing Firebolt OAuth + SQL connection...")
61
+ try:
62
+ engine = sa.create_engine(url)
63
+ with engine.connect() as conn:
64
+ conn.execute(sa.text("SELECT 1"))
65
+ print(" OK: Firebolt credentials accepted.")
66
+ except Exception as exc:
67
+ print(" FAIL: Firebolt authentication or connection failed.")
68
+ print(f" {type(exc).__name__}: {exc}")
69
+ print()
70
+ print("Fix:")
71
+ print(" 1. Firebolt console → Settings → Service accounts")
72
+ print(" 2. Create a new service account or rotate the secret")
73
+ print(" 3. Update FIREBOLT_CLIENT_ID and FIREBOLT_CLIENT_SECRET in .env")
74
+ print(" 4. Confirm FIREBOLT_ACCOUNT_NAME, FIREBOLT_DATABASE, FIREBOLT_ENGINE")
75
+ return 2
76
+
77
+ location_name = s3_location_name_from_env()
78
+ print(f"Testing Firebolt external location '{location_name}'...")
79
+ try:
80
+ engine = sa.create_engine(url)
81
+ with engine.connect() as conn:
82
+ rows = conn.execute(
83
+ sa.text(
84
+ "SELECT 1 FROM information_schema.locations "
85
+ "WHERE location_name = :name"
86
+ ),
87
+ {"name": location_name},
88
+ ).fetchall()
89
+ if not rows:
90
+ print(f" FAIL: Location '{location_name}' not found in this database.")
91
+ print()
92
+ print("Fix:")
93
+ print(" 1. Run CREATE LOCATION in Firebolt for your S3 bucket")
94
+ print(" 2. Set FIREBOLT_S3_LOCATION_NAME to that exact location name")
95
+ return 4
96
+ print(f" OK: Location '{location_name}' exists.")
97
+ except Exception as exc:
98
+ print(f" WARN: Could not verify location ({type(exc).__name__}: {exc})")
99
+
100
+ print()
101
+ print("Testing S3 staging access...")
102
+ try:
103
+ import fsspec
104
+
105
+ fs, _ = fsspec.core.url_to_fs(staging_bucket_url())
106
+ fs.ls(fs._strip_protocol(staging_bucket_url()), detail=False)
107
+ print(" OK: S3 staging bucket reachable.")
108
+ except Exception as exc:
109
+ print(" FAIL: S3 staging not reachable with current AWS credentials.")
110
+ print(f" {type(exc).__name__}: {exc}")
111
+ print()
112
+ print("Fix:")
113
+ print(" export AWS_PROFILE=sandbox")
114
+ print(" aws sso login --profile sandbox")
115
+ return 3
116
+
117
+ print()
118
+ print("All checks passed. You can run phase3/phase4.")
119
+ return 0
120
+
121
+
122
+ if __name__ == "__main__":
123
+ raise SystemExit(main())