frontier-runner 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- frontier_runner-0.1.1/CHANGELOG.md +41 -0
- frontier_runner-0.1.1/LICENSE +21 -0
- frontier_runner-0.1.1/MANIFEST.in +7 -0
- frontier_runner-0.1.1/PKG-INFO +214 -0
- frontier_runner-0.1.1/README.md +160 -0
- frontier_runner-0.1.1/frontier/__init__.py +3 -0
- frontier_runner-0.1.1/frontier/__main__.py +4 -0
- frontier_runner-0.1.1/frontier/adapters/__init__.py +8 -0
- frontier_runner-0.1.1/frontier/adapters/base.py +77 -0
- frontier_runner-0.1.1/frontier/adapters/bigquery.py +118 -0
- frontier_runner-0.1.1/frontier/adapters/databricks.py +90 -0
- frontier_runner-0.1.1/frontier/adapters/postgres.py +93 -0
- frontier_runner-0.1.1/frontier/adapters/redshift.py +89 -0
- frontier_runner-0.1.1/frontier/adapters/snowflake.py +276 -0
- frontier_runner-0.1.1/frontier/api.py +186 -0
- frontier_runner-0.1.1/frontier/artifacts.py +49 -0
- frontier_runner-0.1.1/frontier/cdc/__init__.py +1 -0
- frontier_runner-0.1.1/frontier/cdc/config.py +197 -0
- frontier_runner-0.1.1/frontier/cdc/consume.py +171 -0
- frontier_runner-0.1.1/frontier/cdc/normalize.py +218 -0
- frontier_runner-0.1.1/frontier/cdc/prove.py +586 -0
- frontier_runner-0.1.1/frontier/cdc/route.py +96 -0
- frontier_runner-0.1.1/frontier/cdc/store.py +881 -0
- frontier_runner-0.1.1/frontier/cdc/streams.py +25 -0
- frontier_runner-0.1.1/frontier/cdc/upload.py +416 -0
- frontier_runner-0.1.1/frontier/cli.py +2024 -0
- frontier_runner-0.1.1/frontier/comment.py +354 -0
- frontier_runner-0.1.1/frontier/compare.py +498 -0
- frontier_runner-0.1.1/frontier/config.py +305 -0
- frontier_runner-0.1.1/frontier/credentials.py +239 -0
- frontier_runner-0.1.1/frontier/dbt_artifacts.py +345 -0
- frontier_runner-0.1.1/frontier/errors.py +43 -0
- frontier_runner-0.1.1/frontier/execute.py +848 -0
- frontier_runner-0.1.1/frontier/frontier.py +586 -0
- frontier_runner-0.1.1/frontier/github.py +59 -0
- frontier_runner-0.1.1/frontier/hashing.py +90 -0
- frontier_runner-0.1.1/frontier/impact.py +999 -0
- frontier_runner-0.1.1/frontier/local_config.py +78 -0
- frontier_runner-0.1.1/frontier/onboard/__init__.py +1 -0
- frontier_runner-0.1.1/frontier/onboard/commands.py +593 -0
- frontier_runner-0.1.1/frontier/onboard/constants.py +20 -0
- frontier_runner-0.1.1/frontier/onboard/demo.py +38 -0
- frontier_runner-0.1.1/frontier/onboard/detect.py +130 -0
- frontier_runner-0.1.1/frontier/onboard/discover.py +184 -0
- frontier_runner-0.1.1/frontier/onboard/doctor.py +346 -0
- frontier_runner-0.1.1/frontier/onboard/github.py +273 -0
- frontier_runner-0.1.1/frontier/onboard/gitignore.py +21 -0
- frontier_runner-0.1.1/frontier/onboard/hashkey.py +17 -0
- frontier_runner-0.1.1/frontier/onboard/permissions.py +63 -0
- frontier_runner-0.1.1/frontier/onboard/prompt.py +70 -0
- frontier_runner-0.1.1/frontier/onboard/saas.py +226 -0
- frontier_runner-0.1.1/frontier/onboard/versions.py +20 -0
- frontier_runner-0.1.1/frontier/progress.py +68 -0
- frontier_runner-0.1.1/frontier/proof.py +794 -0
- frontier_runner-0.1.1/frontier/semantic.py +583 -0
- frontier_runner-0.1.1/frontier/snowflake.py +24 -0
- frontier_runner-0.1.1/frontier/snowflake_sql.py +442 -0
- frontier_runner-0.1.1/frontier/sql_fingerprint.py +116 -0
- frontier_runner-0.1.1/frontier/validation.py +211 -0
- frontier_runner-0.1.1/frontier/warehouse.py +390 -0
- frontier_runner-0.1.1/frontier_runner.egg-info/PKG-INFO +214 -0
- frontier_runner-0.1.1/frontier_runner.egg-info/SOURCES.txt +66 -0
- frontier_runner-0.1.1/frontier_runner.egg-info/dependency_links.txt +1 -0
- frontier_runner-0.1.1/frontier_runner.egg-info/entry_points.txt +2 -0
- frontier_runner-0.1.1/frontier_runner.egg-info/requires.txt +10 -0
- frontier_runner-0.1.1/frontier_runner.egg-info/top_level.txt +1 -0
- frontier_runner-0.1.1/pyproject.toml +56 -0
- frontier_runner-0.1.1/setup.cfg +4 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to frontier-runner are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [0.1.1] — 2026-09-06
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- Clean-install SaaS commands resolve stored OS keychain credentials through
|
|
13
|
+
one shared resolver instead of requiring `FRONTIER_API_KEY` in the
|
|
14
|
+
environment.
|
|
15
|
+
- Python package metadata (`requires-python`) matches the supported 3.11–3.13
|
|
16
|
+
installer range.
|
|
17
|
+
- CDC prove tests detect raw entity-ID exposure without treating digit
|
|
18
|
+
sequences inside fingerprints as leaks.
|
|
19
|
+
|
|
20
|
+
### Changed
|
|
21
|
+
|
|
22
|
+
- The release workflow verifies built metadata and smoke-installs the wheel
|
|
23
|
+
(`frontier --version`, `frontier --help`, Snowflake extra) before PyPI
|
|
24
|
+
publish and GitHub Release.
|
|
25
|
+
|
|
26
|
+
## [0.1.0] — 2026-09-06
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
|
|
30
|
+
- Customer CLI for dbt + Snowflake + GitHub assessments.
|
|
31
|
+
- `frontier signup`, `login`, `init`, `discover`, `doctor`, `setup github`,
|
|
32
|
+
`setup hash-key`, `demo change`, `update-check`, `logout`, and `auth status`.
|
|
33
|
+
- Active SaaS semantic-manifest fetch, SQL-change compare/prove, and aggregate upload.
|
|
34
|
+
|
|
35
|
+
### Fixed
|
|
36
|
+
|
|
37
|
+
- SaaS commands resolve stored keychain credentials through one shared
|
|
38
|
+
resolver instead of requiring `FRONTIER_API_KEY` in the environment.
|
|
39
|
+
|
|
40
|
+
[0.1.1]: https://github.com/jadsamara/frontier-runner/compare/v0.1.0...v0.1.1
|
|
41
|
+
[0.1.0]: https://github.com/jadsamara/frontier-runner/releases/tag/v0.1.0
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Frontier
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: frontier-runner
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Customer-side Frontier CLI for dbt + Snowflake + GitHub impact assessments.
|
|
5
|
+
Author: Frontier
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 Frontier
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/jadsamara/frontier-runner
|
|
29
|
+
Project-URL: Changelog, https://github.com/jadsamara/frontier-runner/blob/main/CHANGELOG.md
|
|
30
|
+
Project-URL: Issues, https://github.com/jadsamara/frontier-runner/issues
|
|
31
|
+
Project-URL: Documentation, https://frontier-web-x3l3etwczq-pd.a.run.app/docs/quick-start
|
|
32
|
+
Keywords: dbt,snowflake,github,data
|
|
33
|
+
Classifier: Development Status :: 4 - Beta
|
|
34
|
+
Classifier: Environment :: Console
|
|
35
|
+
Classifier: Intended Audience :: Developers
|
|
36
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
37
|
+
Classifier: Programming Language :: Python :: 3
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
40
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
41
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
42
|
+
Requires-Python: <3.14,>=3.11
|
|
43
|
+
Description-Content-Type: text/markdown
|
|
44
|
+
License-File: LICENSE
|
|
45
|
+
Requires-Dist: pyyaml<7,>=6.0
|
|
46
|
+
Requires-Dist: sqlglot<31,>=30.0
|
|
47
|
+
Requires-Dist: keyring<26,>=25.0
|
|
48
|
+
Provides-Extra: snowflake
|
|
49
|
+
Requires-Dist: snowflake-connector-python<4,>=3.12; extra == "snowflake"
|
|
50
|
+
Provides-Extra: dev
|
|
51
|
+
Requires-Dist: pytest<9,>=8.0; extra == "dev"
|
|
52
|
+
Requires-Dist: build<2,>=1.2; extra == "dev"
|
|
53
|
+
Dynamic: license-file
|
|
54
|
+
|
|
55
|
+
# Frontier Runner
|
|
56
|
+
|
|
57
|
+
Customer-side CLI for dbt + Snowflake + GitHub impact assessments.
|
|
58
|
+
|
|
59
|
+
The runner executes next to the dbt project. It sends metadata and aggregate
|
|
60
|
+
evidence to Frontier SaaS. Warehouse rows and warehouse credentials stay here.
|
|
61
|
+
|
|
62
|
+
Supported stack: **dbt Core or dbt Fusion**, **Snowflake**, **GitHub**, and
|
|
63
|
+
hosted Frontier SaaS. Other warehouses and Git providers are not available in
|
|
64
|
+
this installer.
|
|
65
|
+
|
|
66
|
+
## Install
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pipx install "frontier-runner[snowflake]"
|
|
70
|
+
# or
|
|
71
|
+
python3 -m pip install "frontier-runner[snowflake]"
|
|
72
|
+
frontier --version
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Until the package is on PyPI, install the GitHub Release wheel for a version
|
|
76
|
+
tag. You do not need a commit SHA:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
pipx install \ "frontier-runner[snowflake] @ https://github.com/jadsamara/frontier-runner/releases/download/v0.1.1/frontier_runner-0.1.1-py3-none-any.whl"
|
|
80
|
+
pipx inject frontier-runner "snowflake-connector-python>=3.12,<4"
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## First assessment (under 15 minutes)
|
|
84
|
+
|
|
85
|
+
From the root of an existing dbt project:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
frontier signup
|
|
89
|
+
frontier login --api-key
|
|
90
|
+
frontier init
|
|
91
|
+
frontier discover
|
|
92
|
+
# review and activate the draft in Frontier
|
|
93
|
+
frontier doctor
|
|
94
|
+
frontier setup github
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Commit `.github/workflows/frontier.yml`, open a test pull request, and Frontier
|
|
98
|
+
posts one PR comment. Then open the hosted assessment and confirm it links to
|
|
99
|
+
the pinned semantic manifest.
|
|
100
|
+
|
|
101
|
+
API keys are stored in the OS keychain (or `~/.config/frontier/credentials`
|
|
102
|
+
mode 0600). They are never written to `frontier.yml`, `dbt_project.yml`,
|
|
103
|
+
`profiles.yml`, Git, or generated workflows.
|
|
104
|
+
|
|
105
|
+
## Developer setup
|
|
106
|
+
|
|
107
|
+
From this repository:
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
python3 -m pip install -e ".[dev,snowflake]"
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## Assessment commands
|
|
114
|
+
|
|
115
|
+
`python3 -m frontier` always works after an editable install.
|
|
116
|
+
|
|
117
|
+
`frontier run` reads `~/.dbt/profiles.yml` (and warehouse env vars such as
|
|
118
|
+
`SNOWFLAKE_*`). This installer supports Snowflake. Use `--dry-run` to exercise
|
|
119
|
+
the CLI without a warehouse.
|
|
120
|
+
|
|
121
|
+
Entity IDs in `frontier-run.json` are HMAC-SHA-256 hashed with
|
|
122
|
+
`FRONTIER_ENTITY_HASH_KEY` unless `--include-entity-ids` is set. The key is
|
|
123
|
+
required for hashed output; there is no plain SHA-256 fallback. Rotating the
|
|
124
|
+
key changes entity fingerprints across assessments. The hash key is never sent
|
|
125
|
+
to SaaS.
|
|
126
|
+
|
|
127
|
+
`frontier prove` measures a SQL-change or mutation-repair experiment.
|
|
128
|
+
When `--base-manifest` shows modified, added, or removed SQL, the default
|
|
129
|
+
`seeds/change_events.csv` is ignored: the assessment is the compiled SQL
|
|
130
|
+
diff, not a hand-edited event list. Isolated affected keys are written to
|
|
131
|
+
`DBT_CI.FRONTIER_<run_id>_AFFECTED_KEYS` with separate event and
|
|
132
|
+
SQL-change origins. The M14 impact query runs in Snowflake and is unioned
|
|
133
|
+
for execution. Targeted SQL pushes the key join into source CTEs before
|
|
134
|
+
aggregates. Hand-written `frontier_affected_customers` / repaired models
|
|
135
|
+
are not required for a SQL-change proof. Impact compilation skips models
|
|
136
|
+
tagged `frontier_demo` / `frontier_mutation` and `*_after` overlays unless
|
|
137
|
+
they are the configured target. Candidate discovery never joins
|
|
138
|
+
`frontier_affected_customers` or the isolated keys table; equivalent
|
|
139
|
+
predicates from multiple consumers collapse to one query. When candidates
|
|
140
|
+
exceed `sql_change.rebuild_recommended_pct` of the full entity set
|
|
141
|
+
(default 75, or `FRONTIER_SQL_CHANGE_REBUILD_PCT`), the assessment is
|
|
142
|
+
`FULL_REBUILD_RECOMMENDED` instead of an inefficient targeted proof.
|
|
143
|
+
When base and PR SQL differ, a
|
|
144
|
+
missing or failed impact query is `FULL_REBUILD_REQUIRED` rather than an
|
|
145
|
+
event-only frontier. Customer CI must call `prove`, not `run`.
|
|
146
|
+
|
|
147
|
+
`frontier record-failure` writes a failed assessment without reading
|
|
148
|
+
`target/manifest.json` or `run_results.json`. Use it when dbt build fails so CI
|
|
149
|
+
cannot upload stale artifacts.
|
|
150
|
+
|
|
151
|
+
`frontier cdc inspect|status|consume|prove|upload` reads `frontier-cdc.yml` in
|
|
152
|
+
the dbt project. Inspect prints stream mappings. Status calls
|
|
153
|
+
`SYSTEM$STREAM_HAS_DATA` without consuming. Consume copies pending Snowflake
|
|
154
|
+
stream rows into `DATA_AGENT_DEV.FRONTIER_CDC` control tables inside a
|
|
155
|
+
transaction, then normalizes a DELETE/INSERT `METADATA$ISUPDATE=TRUE` pair
|
|
156
|
+
into one UPDATE. A plain SELECT is never treated as consumption. `cdc prove`
|
|
157
|
+
claims the oldest CAPTURED or FAILED batch, routes events to target keys from
|
|
158
|
+
the YAML mapping, materializes `DBT_DEV.FRONTIER_<batch_id>_AFFECTED_KEYS`,
|
|
159
|
+
and runs targeted compiled `customer_summary` SQL against current source
|
|
160
|
+
state. The existing mart is the pre-change baseline. Completion requires
|
|
161
|
+
routing and validation, not merely stream consumption. A candidate no-op is a
|
|
162
|
+
successful conservative assessment. Default prove is assessment-only;
|
|
163
|
+
`--apply` is required to mutate the mart. `cdc upload` sends aggregate CDC
|
|
164
|
+
evidence to SaaS without recapturing or reproving. If a previous COMPLETED
|
|
165
|
+
batch was not applied, a later prove fails with `BASELINE_STALE`. Logs include
|
|
166
|
+
stream name, batch id, counts, status, and duration — never entity IDs, row
|
|
167
|
+
contents, or credentials. Do not upload raw CDC keys to SaaS. Scheduled CDC
|
|
168
|
+
processing is not part of this installer.
|
|
169
|
+
|
|
170
|
+
`frontier compare` reads compiled SQL from the base-branch and PR manifests
|
|
171
|
+
(and `target/compiled` / `target-base/compiled` when `compiled_code` is
|
|
172
|
+
missing), classifies semantic changes with a restricted Snowflake parser
|
|
173
|
+
(sqlglot), and compiles supported diffs into a candidate-key impact query.
|
|
174
|
+
Alias and formatting changes are ignored. Grain changes, unknown UDFs, empty
|
|
175
|
+
compiled SQL, and other unsupported SQL return `FULL_REBUILD_REQUIRED`
|
|
176
|
+
instead of an empty candidate set. `frontier prove` confirms and repairs using
|
|
177
|
+
the production model whose compiled SQL actually changed, not a downstream
|
|
178
|
+
mart that only `ref()`s it. The comparison does not send warehouse rows to
|
|
179
|
+
SaaS. `inspect`, `run`, and `prove` accept `--base-manifest` so artifact
|
|
180
|
+
fingerprints, change kinds, and impact status are stored on the uploaded
|
|
181
|
+
assessment.
|
|
182
|
+
|
|
183
|
+
`frontier upload` posts `target/frontier-run.json` to `POST /api/v1/runs`. It
|
|
184
|
+
retries HTTP 429/5xx and network errors, and honors `Retry-After`. SaaS
|
|
185
|
+
commands resolve credentials in this order: `FRONTIER_API_KEY`, the OS
|
|
186
|
+
keychain, the `0600` fallback file, then `FRONTIER_DEMO_API_KEY` only when
|
|
187
|
+
`FRONTIER_ALLOW_LOCAL_MANIFEST` is set outside GitHub Actions. If none are
|
|
188
|
+
present, the CLI exits with `AUTH_REQUIRED: Run \`frontier login --api-key\``.
|
|
189
|
+
Hashed uploads set `entityIdsHashed: true`.
|
|
190
|
+
|
|
191
|
+
In GitHub Actions, assessments use `{project}-{GITHUB_SHA}` as `externalRunId`
|
|
192
|
+
and record repository, branch, commit, and PR number. After a successful
|
|
193
|
+
upload the runner upserts one pull-request comment (aggregates only, plus a
|
|
194
|
+
dashboard `/runs/<id>` link). `GITHUB_TOKEN` stays in the customer job.
|
|
195
|
+
`FRONTIER_DRY_RUN=true` is only for the SaaS fixture self-test and is rejected
|
|
196
|
+
by `frontier prove` in GitHub Actions. Customer CI must execute against the
|
|
197
|
+
live warehouse. Uploaded assessments set `runMode` to `live` or `fixture`.
|
|
198
|
+
`frontier upload --blocking` (or `FRONTIER_BLOCKING=true`) uploads and comments
|
|
199
|
+
first, then exits 1 if the assessment failed. The generated first-verification
|
|
200
|
+
workflow uses `FRONTIER_BLOCKING=false` unless you pass `--blocking`.
|
|
201
|
+
|
|
202
|
+
## Releases
|
|
203
|
+
|
|
204
|
+
Pin an immutable released version:
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
pip install "frontier-runner[snowflake]==0.1.1"
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Until PyPI trusted publishing is reviewed and live, install the GitHub Release
|
|
211
|
+
wheel for the same version tag. Do not look up a runner Git SHA.
|
|
212
|
+
|
|
213
|
+
Do not `pip install ./runner` from a dbt repository. That path exists only in
|
|
214
|
+
the SaaS monorepo.
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# Frontier Runner
|
|
2
|
+
|
|
3
|
+
Customer-side CLI for dbt + Snowflake + GitHub impact assessments.
|
|
4
|
+
|
|
5
|
+
The runner executes next to the dbt project. It sends metadata and aggregate
|
|
6
|
+
evidence to Frontier SaaS. Warehouse rows and warehouse credentials stay here.
|
|
7
|
+
|
|
8
|
+
Supported stack: **dbt Core or dbt Fusion**, **Snowflake**, **GitHub**, and
|
|
9
|
+
hosted Frontier SaaS. Other warehouses and Git providers are not available in
|
|
10
|
+
this installer.
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pipx install "frontier-runner[snowflake]"
|
|
16
|
+
# or
|
|
17
|
+
python3 -m pip install "frontier-runner[snowflake]"
|
|
18
|
+
frontier --version
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Until the package is on PyPI, install the GitHub Release wheel for a version
|
|
22
|
+
tag. You do not need a commit SHA:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pipx install \ "frontier-runner[snowflake] @ https://github.com/jadsamara/frontier-runner/releases/download/v0.1.1/frontier_runner-0.1.1-py3-none-any.whl"
|
|
26
|
+
pipx inject frontier-runner "snowflake-connector-python>=3.12,<4"
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## First assessment (under 15 minutes)
|
|
30
|
+
|
|
31
|
+
From the root of an existing dbt project:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
frontier signup
|
|
35
|
+
frontier login --api-key
|
|
36
|
+
frontier init
|
|
37
|
+
frontier discover
|
|
38
|
+
# review and activate the draft in Frontier
|
|
39
|
+
frontier doctor
|
|
40
|
+
frontier setup github
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Commit `.github/workflows/frontier.yml`, open a test pull request, and Frontier
|
|
44
|
+
posts one PR comment. Then open the hosted assessment and confirm it links to
|
|
45
|
+
the pinned semantic manifest.
|
|
46
|
+
|
|
47
|
+
API keys are stored in the OS keychain (or `~/.config/frontier/credentials`
|
|
48
|
+
mode 0600). They are never written to `frontier.yml`, `dbt_project.yml`,
|
|
49
|
+
`profiles.yml`, Git, or generated workflows.
|
|
50
|
+
|
|
51
|
+
## Developer setup
|
|
52
|
+
|
|
53
|
+
From this repository:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
python3 -m pip install -e ".[dev,snowflake]"
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Assessment commands
|
|
60
|
+
|
|
61
|
+
`python3 -m frontier` always works after an editable install.
|
|
62
|
+
|
|
63
|
+
`frontier run` reads `~/.dbt/profiles.yml` (and warehouse env vars such as
|
|
64
|
+
`SNOWFLAKE_*`). This installer supports Snowflake. Use `--dry-run` to exercise
|
|
65
|
+
the CLI without a warehouse.
|
|
66
|
+
|
|
67
|
+
Entity IDs in `frontier-run.json` are HMAC-SHA-256 hashed with
|
|
68
|
+
`FRONTIER_ENTITY_HASH_KEY` unless `--include-entity-ids` is set. The key is
|
|
69
|
+
required for hashed output; there is no plain SHA-256 fallback. Rotating the
|
|
70
|
+
key changes entity fingerprints across assessments. The hash key is never sent
|
|
71
|
+
to SaaS.
|
|
72
|
+
|
|
73
|
+
`frontier prove` measures a SQL-change or mutation-repair experiment.
|
|
74
|
+
When `--base-manifest` shows modified, added, or removed SQL, the default
|
|
75
|
+
`seeds/change_events.csv` is ignored: the assessment is the compiled SQL
|
|
76
|
+
diff, not a hand-edited event list. Isolated affected keys are written to
|
|
77
|
+
`DBT_CI.FRONTIER_<run_id>_AFFECTED_KEYS` with separate event and
|
|
78
|
+
SQL-change origins. The M14 impact query runs in Snowflake and is unioned
|
|
79
|
+
for execution. Targeted SQL pushes the key join into source CTEs before
|
|
80
|
+
aggregates. Hand-written `frontier_affected_customers` / repaired models
|
|
81
|
+
are not required for a SQL-change proof. Impact compilation skips models
|
|
82
|
+
tagged `frontier_demo` / `frontier_mutation` and `*_after` overlays unless
|
|
83
|
+
they are the configured target. Candidate discovery never joins
|
|
84
|
+
`frontier_affected_customers` or the isolated keys table; equivalent
|
|
85
|
+
predicates from multiple consumers collapse to one query. When candidates
|
|
86
|
+
exceed `sql_change.rebuild_recommended_pct` of the full entity set
|
|
87
|
+
(default 75, or `FRONTIER_SQL_CHANGE_REBUILD_PCT`), the assessment is
|
|
88
|
+
`FULL_REBUILD_RECOMMENDED` instead of an inefficient targeted proof.
|
|
89
|
+
When base and PR SQL differ, a
|
|
90
|
+
missing or failed impact query is `FULL_REBUILD_REQUIRED` rather than an
|
|
91
|
+
event-only frontier. Customer CI must call `prove`, not `run`.
|
|
92
|
+
|
|
93
|
+
`frontier record-failure` writes a failed assessment without reading
|
|
94
|
+
`target/manifest.json` or `run_results.json`. Use it when dbt build fails so CI
|
|
95
|
+
cannot upload stale artifacts.
|
|
96
|
+
|
|
97
|
+
`frontier cdc inspect|status|consume|prove|upload` reads `frontier-cdc.yml` in
|
|
98
|
+
the dbt project. Inspect prints stream mappings. Status calls
|
|
99
|
+
`SYSTEM$STREAM_HAS_DATA` without consuming. Consume copies pending Snowflake
|
|
100
|
+
stream rows into `DATA_AGENT_DEV.FRONTIER_CDC` control tables inside a
|
|
101
|
+
transaction, then normalizes a DELETE/INSERT `METADATA$ISUPDATE=TRUE` pair
|
|
102
|
+
into one UPDATE. A plain SELECT is never treated as consumption. `cdc prove`
|
|
103
|
+
claims the oldest CAPTURED or FAILED batch, routes events to target keys from
|
|
104
|
+
the YAML mapping, materializes `DBT_DEV.FRONTIER_<batch_id>_AFFECTED_KEYS`,
|
|
105
|
+
and runs targeted compiled `customer_summary` SQL against current source
|
|
106
|
+
state. The existing mart is the pre-change baseline. Completion requires
|
|
107
|
+
routing and validation, not merely stream consumption. A candidate no-op is a
|
|
108
|
+
successful conservative assessment. Default prove is assessment-only;
|
|
109
|
+
`--apply` is required to mutate the mart. `cdc upload` sends aggregate CDC
|
|
110
|
+
evidence to SaaS without recapturing or reproving. If a previous COMPLETED
|
|
111
|
+
batch was not applied, a later prove fails with `BASELINE_STALE`. Logs include
|
|
112
|
+
stream name, batch id, counts, status, and duration — never entity IDs, row
|
|
113
|
+
contents, or credentials. Do not upload raw CDC keys to SaaS. Scheduled CDC
|
|
114
|
+
processing is not part of this installer.
|
|
115
|
+
|
|
116
|
+
`frontier compare` reads compiled SQL from the base-branch and PR manifests
|
|
117
|
+
(and `target/compiled` / `target-base/compiled` when `compiled_code` is
|
|
118
|
+
missing), classifies semantic changes with a restricted Snowflake parser
|
|
119
|
+
(sqlglot), and compiles supported diffs into a candidate-key impact query.
|
|
120
|
+
Alias and formatting changes are ignored. Grain changes, unknown UDFs, empty
|
|
121
|
+
compiled SQL, and other unsupported SQL return `FULL_REBUILD_REQUIRED`
|
|
122
|
+
instead of an empty candidate set. `frontier prove` confirms and repairs using
|
|
123
|
+
the production model whose compiled SQL actually changed, not a downstream
|
|
124
|
+
mart that only `ref()`s it. The comparison does not send warehouse rows to
|
|
125
|
+
SaaS. `inspect`, `run`, and `prove` accept `--base-manifest` so artifact
|
|
126
|
+
fingerprints, change kinds, and impact status are stored on the uploaded
|
|
127
|
+
assessment.
|
|
128
|
+
|
|
129
|
+
`frontier upload` posts `target/frontier-run.json` to `POST /api/v1/runs`. It
|
|
130
|
+
retries HTTP 429/5xx and network errors, and honors `Retry-After`. SaaS
|
|
131
|
+
commands resolve credentials in this order: `FRONTIER_API_KEY`, the OS
|
|
132
|
+
keychain, the `0600` fallback file, then `FRONTIER_DEMO_API_KEY` only when
|
|
133
|
+
`FRONTIER_ALLOW_LOCAL_MANIFEST` is set outside GitHub Actions. If none are
|
|
134
|
+
present, the CLI exits with `AUTH_REQUIRED: Run \`frontier login --api-key\``.
|
|
135
|
+
Hashed uploads set `entityIdsHashed: true`.
|
|
136
|
+
|
|
137
|
+
In GitHub Actions, assessments use `{project}-{GITHUB_SHA}` as `externalRunId`
|
|
138
|
+
and record repository, branch, commit, and PR number. After a successful
|
|
139
|
+
upload the runner upserts one pull-request comment (aggregates only, plus a
|
|
140
|
+
dashboard `/runs/<id>` link). `GITHUB_TOKEN` stays in the customer job.
|
|
141
|
+
`FRONTIER_DRY_RUN=true` is only for the SaaS fixture self-test and is rejected
|
|
142
|
+
by `frontier prove` in GitHub Actions. Customer CI must execute against the
|
|
143
|
+
live warehouse. Uploaded assessments set `runMode` to `live` or `fixture`.
|
|
144
|
+
`frontier upload --blocking` (or `FRONTIER_BLOCKING=true`) uploads and comments
|
|
145
|
+
first, then exits 1 if the assessment failed. The generated first-verification
|
|
146
|
+
workflow uses `FRONTIER_BLOCKING=false` unless you pass `--blocking`.
|
|
147
|
+
|
|
148
|
+
## Releases
|
|
149
|
+
|
|
150
|
+
Pin an immutable released version:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
pip install "frontier-runner[snowflake]==0.1.1"
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Until PyPI trusted publishing is reviewed and live, install the GitHub Release
|
|
157
|
+
wheel for the same version tag. Do not look up a runner Git SHA.
|
|
158
|
+
|
|
159
|
+
Do not `pip install ./runner` from a dbt repository. That path exists only in
|
|
160
|
+
the SaaS monorepo.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from frontier.config import ConfigError
|
|
6
|
+
from frontier.warehouse import (
|
|
7
|
+
sql_string,
|
|
8
|
+
split_relation_parts,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class CursorAdapter:
|
|
13
|
+
"""Shared execute/close for DB-API style connections."""
|
|
14
|
+
|
|
15
|
+
warehouse_type: str
|
|
16
|
+
dialect: str
|
|
17
|
+
_connection: Any = None
|
|
18
|
+
last_query_id: str | None = None
|
|
19
|
+
|
|
20
|
+
def execute(self, sql: str) -> list[tuple[Any, ...]]:
|
|
21
|
+
connection = self._require_connection()
|
|
22
|
+
cursor = connection.cursor()
|
|
23
|
+
try:
|
|
24
|
+
cursor.execute(sql)
|
|
25
|
+
self.last_query_id = getattr(cursor, "sfqid", None) or getattr(cursor, "sfqId", None)
|
|
26
|
+
if self.last_query_id is not None:
|
|
27
|
+
self.last_query_id = str(self.last_query_id)
|
|
28
|
+
if cursor.description is None:
|
|
29
|
+
return []
|
|
30
|
+
rows = cursor.fetchall() or []
|
|
31
|
+
return [tuple(row) for row in rows]
|
|
32
|
+
finally:
|
|
33
|
+
cursor.close()
|
|
34
|
+
|
|
35
|
+
def get_query_profile(self, query_id: str) -> dict[str, Any]:
|
|
36
|
+
del query_id
|
|
37
|
+
return {}
|
|
38
|
+
|
|
39
|
+
def close(self) -> None:
|
|
40
|
+
connection = self._connection
|
|
41
|
+
if connection is not None:
|
|
42
|
+
connection.close()
|
|
43
|
+
self._connection = None
|
|
44
|
+
|
|
45
|
+
def _require_connection(self) -> Any:
|
|
46
|
+
if self._connection is None:
|
|
47
|
+
raise ConfigError(f"{self.warehouse_type} adapter is not connected")
|
|
48
|
+
return self._connection
|
|
49
|
+
|
|
50
|
+
def _table_lookup_sql(self, relation: str) -> str:
|
|
51
|
+
catalog, schema, table = split_relation_parts(relation)
|
|
52
|
+
sql = (
|
|
53
|
+
"select 1 as present from information_schema.tables "
|
|
54
|
+
f"where lower(table_name) = lower({sql_string(table)})"
|
|
55
|
+
)
|
|
56
|
+
if schema:
|
|
57
|
+
sql += f" and lower(table_schema) = lower({sql_string(schema)})"
|
|
58
|
+
if catalog:
|
|
59
|
+
sql += f" and lower(table_catalog) = lower({sql_string(catalog)})"
|
|
60
|
+
return sql + " limit 1"
|
|
61
|
+
|
|
62
|
+
def relation_exists(self, relation: str) -> bool:
|
|
63
|
+
return bool(self.execute(self._table_lookup_sql(relation)))
|
|
64
|
+
|
|
65
|
+
def estimate_query_cost(self, sql: str) -> dict[str, Any]:
|
|
66
|
+
try:
|
|
67
|
+
rows = self.execute(f"explain {sql}")
|
|
68
|
+
except Exception:
|
|
69
|
+
return {"estimated": False, "warehouse_type": self.warehouse_type}
|
|
70
|
+
return {
|
|
71
|
+
"estimated": True,
|
|
72
|
+
"warehouse_type": self.warehouse_type,
|
|
73
|
+
"plan_rows": len(rows),
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
def get_query_history(self, run_id: str) -> list[dict[str, Any]]:
|
|
77
|
+
return []
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from frontier.config import ConfigError, redact
|
|
6
|
+
from frontier.warehouse import (
|
|
7
|
+
env_value,
|
|
8
|
+
quote_identifier,
|
|
9
|
+
split_relation_parts,
|
|
10
|
+
sql_string,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class BigQueryAdapter:
|
|
15
|
+
warehouse_type = "bigquery"
|
|
16
|
+
dialect = "bigquery"
|
|
17
|
+
|
|
18
|
+
def __init__(self, client: Any | None = None, *, project: str | None = None, dataset: str | None = None):
|
|
19
|
+
self._client = client
|
|
20
|
+
self.project = project
|
|
21
|
+
self.dataset = dataset
|
|
22
|
+
|
|
23
|
+
def quote_identifier(self, value: str) -> str:
|
|
24
|
+
return quote_identifier(value, "`")
|
|
25
|
+
|
|
26
|
+
def execute(self, sql: str) -> list[tuple[Any, ...]]:
|
|
27
|
+
client = self._require_client()
|
|
28
|
+
rows = list(client.query(sql).result())
|
|
29
|
+
return [tuple(row.values()) for row in rows]
|
|
30
|
+
|
|
31
|
+
def relation_exists(self, relation: str) -> bool:
|
|
32
|
+
catalog, schema, table = split_relation_parts(relation)
|
|
33
|
+
project = catalog or self.project
|
|
34
|
+
dataset = schema or self.dataset
|
|
35
|
+
if not project or not dataset:
|
|
36
|
+
raise ConfigError("BigQuery relation_exists needs project and dataset")
|
|
37
|
+
sql = (
|
|
38
|
+
f"select 1 from `{project}.{dataset}.INFORMATION_SCHEMA.TABLES` "
|
|
39
|
+
f"where lower(table_name) = lower({sql_string(table)}) "
|
|
40
|
+
"limit 1"
|
|
41
|
+
)
|
|
42
|
+
return bool(self.execute(sql))
|
|
43
|
+
|
|
44
|
+
def estimate_query_cost(self, sql: str) -> dict[str, Any]:
|
|
45
|
+
client = self._require_client()
|
|
46
|
+
job_config = _dry_run_config()
|
|
47
|
+
job = client.query(sql, job_config=job_config)
|
|
48
|
+
bytes_processed = int(getattr(job, "total_bytes_processed", 0) or 0)
|
|
49
|
+
return {
|
|
50
|
+
"estimated": True,
|
|
51
|
+
"warehouse_type": self.warehouse_type,
|
|
52
|
+
"total_bytes_processed": bytes_processed,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
def get_query_history(self, run_id: str) -> list[dict[str, Any]]:
|
|
56
|
+
project = self.project
|
|
57
|
+
if not project:
|
|
58
|
+
return []
|
|
59
|
+
tagged = sql_string(run_id)
|
|
60
|
+
sql = (
|
|
61
|
+
f"select job_id, state, total_bytes_processed "
|
|
62
|
+
f"from `{project}.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` "
|
|
63
|
+
f"where query like '%' || {tagged} || '%' "
|
|
64
|
+
"order by creation_time desc limit 50"
|
|
65
|
+
)
|
|
66
|
+
try:
|
|
67
|
+
rows = self.execute(sql)
|
|
68
|
+
except Exception:
|
|
69
|
+
return []
|
|
70
|
+
return [
|
|
71
|
+
{"query_id": row[0], "status": row[1], "bytes": row[2] if len(row) > 2 else None}
|
|
72
|
+
for row in rows
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
def close(self) -> None:
|
|
76
|
+
client = self._client
|
|
77
|
+
if client is not None and hasattr(client, "close"):
|
|
78
|
+
client.close()
|
|
79
|
+
self._client = None
|
|
80
|
+
|
|
81
|
+
def describe(self) -> dict[str, Any]:
|
|
82
|
+
return redact(
|
|
83
|
+
{
|
|
84
|
+
"warehouse_type": self.warehouse_type,
|
|
85
|
+
"project": self.project,
|
|
86
|
+
"dataset": self.dataset,
|
|
87
|
+
}
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
def _require_client(self) -> Any:
|
|
91
|
+
if self._client is None:
|
|
92
|
+
raise ConfigError("bigquery adapter is not connected")
|
|
93
|
+
return self._client
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _dry_run_config() -> Any:
|
|
97
|
+
from google.cloud import bigquery
|
|
98
|
+
|
|
99
|
+
return bigquery.QueryJobConfig(dry_run=True, use_query_cache=False)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def open_bigquery_adapter(settings: dict[str, Any]) -> BigQueryAdapter:
|
|
103
|
+
try:
|
|
104
|
+
from google.cloud import bigquery
|
|
105
|
+
except ImportError as error:
|
|
106
|
+
raise ConfigError(
|
|
107
|
+
"Install frontier-runner[bigquery] to open a BigQuery session",
|
|
108
|
+
) from error
|
|
109
|
+
project = env_value("BIGQUERY_PROJECT", "GOOGLE_CLOUD_PROJECT") or settings.get("project")
|
|
110
|
+
dataset = (
|
|
111
|
+
env_value("BIGQUERY_DATASET")
|
|
112
|
+
or settings.get("dataset")
|
|
113
|
+
or settings.get("schema")
|
|
114
|
+
)
|
|
115
|
+
if not project:
|
|
116
|
+
raise ConfigError("BigQuery project is required (BIGQUERY_PROJECT or dbt profile)")
|
|
117
|
+
client = bigquery.Client(project=str(project))
|
|
118
|
+
return BigQueryAdapter(client, project=str(project), dataset=str(dataset) if dataset else None)
|