governor-audit 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- governor_audit-0.0.1/PKG-INFO +94 -0
- governor_audit-0.0.1/README.md +57 -0
- governor_audit-0.0.1/pyproject.toml +65 -0
- governor_audit-0.0.1/setup.cfg +4 -0
- governor_audit-0.0.1/src/governor_audit/__init__.py +20 -0
- governor_audit-0.0.1/src/governor_audit/auth/__init__.py +0 -0
- governor_audit-0.0.1/src/governor_audit/auth/adc.py +158 -0
- governor_audit-0.0.1/src/governor_audit/cli/__init__.py +0 -0
- governor_audit-0.0.1/src/governor_audit/cli/app.py +55 -0
- governor_audit-0.0.1/src/governor_audit/cli/commands/__init__.py +0 -0
- governor_audit-0.0.1/src/governor_audit/cli/commands/init.py +165 -0
- governor_audit-0.0.1/src/governor_audit/cli/commands/scan.py +125 -0
- governor_audit-0.0.1/src/governor_audit/cli/commands/status.py +62 -0
- governor_audit-0.0.1/src/governor_audit/config/__init__.py +0 -0
- governor_audit-0.0.1/src/governor_audit/config/loader.py +133 -0
- governor_audit-0.0.1/src/governor_audit/config/schema.py +181 -0
- governor_audit-0.0.1/src/governor_audit/db/__init__.py +0 -0
- governor_audit-0.0.1/src/governor_audit/db/base.py +47 -0
- governor_audit-0.0.1/src/governor_audit/db/migrate.py +34 -0
- governor_audit-0.0.1/src/governor_audit/db/models.py +183 -0
- governor_audit-0.0.1/src/governor_audit/db/session.py +90 -0
- governor_audit-0.0.1/src/governor_audit/scan/__init__.py +0 -0
- governor_audit-0.0.1/src/governor_audit/scan/bigquery_client.py +75 -0
- governor_audit-0.0.1/src/governor_audit/scan/detection.py +201 -0
- governor_audit-0.0.1/src/governor_audit/scan/information_schema.py +258 -0
- governor_audit-0.0.1/src/governor_audit/scan/lock.py +90 -0
- governor_audit-0.0.1/src/governor_audit/scan/orchestrator.py +269 -0
- governor_audit-0.0.1/src/governor_audit/scan/persistence.py +167 -0
- governor_audit-0.0.1/src/governor_audit/scan/sentinels.py +153 -0
- governor_audit-0.0.1/src/governor_audit/version.py +11 -0
- governor_audit-0.0.1/src/governor_audit/web/__init__.py +0 -0
- governor_audit-0.0.1/src/governor_audit/web/main.py +125 -0
- governor_audit-0.0.1/src/governor_audit/web/routes/__init__.py +0 -0
- governor_audit-0.0.1/src/governor_audit/web/routes/configurations.py +525 -0
- governor_audit-0.0.1/src/governor_audit/web/routes/findings.py +591 -0
- governor_audit-0.0.1/src/governor_audit/web/routes/scan.py +146 -0
- governor_audit-0.0.1/src/governor_audit/web/routes/settings.py +272 -0
- governor_audit-0.0.1/src/governor_audit/web/routes/status.py +77 -0
- governor_audit-0.0.1/src/governor_audit/web/static/css/app.css +2 -0
- governor_audit-0.0.1/src/governor_audit/web/static/js/chart-utils.js +180 -0
- governor_audit-0.0.1/src/governor_audit/web/template_filters.py +44 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/configurations/detail.html +302 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/configurations/new.html +134 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/dashboard/_cost_drivers.html +134 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/dashboard/_cost_summary.html +80 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/dashboard/_opportunities_summary.html +43 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/dashboard/_pagination.html +37 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/dashboard/_run_scan_form.html +59 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/dashboard/_trend_chart.html +105 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/dashboard/index.html +40 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/dashboard/job_detail.html +275 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/layout.html +153 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/opportunities/index.html +23 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/settings/appearance.html +71 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/settings/detection_rules.html +59 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/settings/index.html +142 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/settings/layout.html +41 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/settings/llm.html +133 -0
- governor_audit-0.0.1/src/governor_audit/web/templates/settings/profile.html +56 -0
- governor_audit-0.0.1/src/governor_audit.egg-info/PKG-INFO +94 -0
- governor_audit-0.0.1/src/governor_audit.egg-info/SOURCES.txt +63 -0
- governor_audit-0.0.1/src/governor_audit.egg-info/dependency_links.txt +1 -0
- governor_audit-0.0.1/src/governor_audit.egg-info/entry_points.txt +2 -0
- governor_audit-0.0.1/src/governor_audit.egg-info/requires.txt +9 -0
- governor_audit-0.0.1/src/governor_audit.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: governor-audit
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Read-only BigQuery cost-audit tool — single-user, gcloud ADC only, no GCS / no GitHub / no dbt installation.
|
|
5
|
+
Author-email: Simple Machines <hello@simplemachines.co.nz>
|
|
6
|
+
Maintainer-email: Simple Machines <hello@simplemachines.co.nz>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/simple-machines/governor
|
|
9
|
+
Project-URL: Repository, https://github.com/simple-machines/governor
|
|
10
|
+
Project-URL: Issues, https://github.com/simple-machines/governor/issues
|
|
11
|
+
Project-URL: Documentation, https://github.com/simple-machines/governor/tree/main/specs/141-production-audit
|
|
12
|
+
Project-URL: Changelog, https://github.com/simple-machines/governor/releases?q=audit-
|
|
13
|
+
Keywords: bigquery,cost-optimization,dbt,audit,data-engineering,warehouse
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Environment :: Console
|
|
16
|
+
Classifier: Environment :: Web Environment
|
|
17
|
+
Classifier: Intended Audience :: Developers
|
|
18
|
+
Classifier: Intended Audience :: System Administrators
|
|
19
|
+
Classifier: Operating System :: OS Independent
|
|
20
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Topic :: Database
|
|
23
|
+
Classifier: Topic :: Office/Business :: Financial
|
|
24
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
25
|
+
Classifier: Topic :: System :: Monitoring
|
|
26
|
+
Requires-Python: >=3.14
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
Requires-Dist: governor-core==0.7.29
|
|
29
|
+
Requires-Dist: typer>=0.12.0
|
|
30
|
+
Requires-Dist: rich>=13.0.0
|
|
31
|
+
Requires-Dist: fastapi>=0.115.0
|
|
32
|
+
Requires-Dist: jinja2>=3.1.0
|
|
33
|
+
Requires-Dist: uvicorn>=0.30.6
|
|
34
|
+
Requires-Dist: pydantic>=2.0.0
|
|
35
|
+
Requires-Dist: google-cloud-bigquery>=3.20.0
|
|
36
|
+
Requires-Dist: google-auth>=2.30.0
|
|
37
|
+
|
|
38
|
+
# governor-audit
|
|
39
|
+
|
|
40
|
+
Read-only BigQuery cost-audit tool for single-user production audits.
|
|
41
|
+
|
|
42
|
+
> **Posture**: Single-user. gcloud ADC only. No GCS, no GitHub, no service-account JSON, no dbt installation, no shadow validation. The only thing it talks to over the network is BigQuery — and only to query `INFORMATION_SCHEMA.JOBS_BY_PROJECT`.
|
|
43
|
+
|
|
44
|
+
## When to use this vs. the other governor packages
|
|
45
|
+
|
|
46
|
+
- **`governor-audit`** (this package): you have read access to a prod BigQuery project. You want a fast cost audit + detection findings without touching the dbt source code, running dbt, or setting up cloud infrastructure.
|
|
47
|
+
- **`governor-cli`**: you have the dbt project source on your machine and want to run dbt + propose fixes locally.
|
|
48
|
+
- **`governor-web`**: you operate the platform; you want shared infrastructure (GCS-backed manifests, GitHub PRs, scheduled syncs) for a team.
|
|
49
|
+
|
|
50
|
+
## What you get
|
|
51
|
+
|
|
52
|
+
- **Dashboard** — Total / Build / Consumption / Flagged spend KPI cards, top-20 spenders bar chart, paginated cost-drivers table with click-to-sort columns and a per-row issue count.
|
|
53
|
+
- **Detection engine** — every enabled rule from `governor_core.opportunities.rules` (partition_pruning, shuffle_spill, slot_contention, join_explosion, dead_cte, dead_column, dead_window_expression, redundant_order_by, unused_aggregation_output, unused_join) runs on each cached job; opportunities feed the Active Issues pane.
|
|
54
|
+
- **Job detail** — full SQL viewer with expand / copy buttons, metadata panes, and the list of detection-rule findings for that job.
|
|
55
|
+
- **Settings** — Account (gcloud principal + ADC probe), Appearance (light / dark / system), AI / LLM (provider, model, params — reviewer code lands in v0.0.2), Detection Rules (per-rule enable/disable).
|
|
56
|
+
- **Reset cache** — wipe everything `INFORMATION_SCHEMA` collected without losing your config.
|
|
57
|
+
|
|
58
|
+
## Quickstart
|
|
59
|
+
|
|
60
|
+
```sh
|
|
61
|
+
gcloud auth application-default login
|
|
62
|
+
uv tool install governor-audit
|
|
63
|
+
governor-audit init --project prod-warehouse-123 --region us
|
|
64
|
+
governor-audit scan --days 30
|
|
65
|
+
governor-audit start
|
|
66
|
+
# open http://localhost:8765
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The web UI exposes the same actions as the CLI plus the dashboard / settings views. After the first `init` you can do everything from the browser.
|
|
70
|
+
|
|
71
|
+
See [the spec's quickstart](../../specs/141-production-audit/quickstart.md) for the full first-audit walkthrough.
|
|
72
|
+
|
|
73
|
+
## Architecture
|
|
74
|
+
|
|
75
|
+
- **Storage**: SQLite at `~/.governor-audit/state.db` via `governor_core.db.sqlite_compat`.
|
|
76
|
+
- **Auth**: gcloud Application Default Credentials only — `google.auth.default()`. No service-account JSON. No browser OAuth.
|
|
77
|
+
- **Workload classification**: manifest-free heuristic — dbt-originated CTAS / MERGE / INSERT / UPDATE / DELETE → `build`; non-dbt SELECT → `consumption`; ambiguous → `other`. Driven by the `/* {"app": "dbt"` comment-prefix the dbt-bigquery adapter prepends.
|
|
78
|
+
- **Loopback only**: the FastAPI app rejects any request whose `Host:` header isn't a localhost variant. Not a public service.
|
|
79
|
+
|
|
80
|
+
See [spec 141](../../specs/141-production-audit/) for the complete spec set:
|
|
81
|
+
|
|
82
|
+
- [spec.md](../../specs/141-production-audit/spec.md) — feature spec
|
|
83
|
+
- [plan.md](../../specs/141-production-audit/plan.md) — implementation plan
|
|
84
|
+
- [data-model.md](../../specs/141-production-audit/data-model.md) — schemas
|
|
85
|
+
- [contracts/cli.md](../../specs/141-production-audit/contracts/cli.md) — CLI contract
|
|
86
|
+
- [contracts/web-routes.md](../../specs/141-production-audit/contracts/web-routes.md) — web routes contract
|
|
87
|
+
|
|
88
|
+
## Versioning
|
|
89
|
+
|
|
90
|
+
`governor-audit` ships on its own version track, decoupled from the cloud bundle (`governor-core` / `governor-web` / `governor-cli` / `governor-bq`). Audit `v0.0.1` and cloud `v0.7.x` coexist. See [scripts/release-audit.sh](../../scripts/release-audit.sh) for the release flow.
|
|
91
|
+
|
|
92
|
+
## License
|
|
93
|
+
|
|
94
|
+
MIT.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# governor-audit
|
|
2
|
+
|
|
3
|
+
Read-only BigQuery cost-audit tool for single-user production audits.
|
|
4
|
+
|
|
5
|
+
> **Posture**: Single-user. gcloud ADC only. No GCS, no GitHub, no service-account JSON, no dbt installation, no shadow validation. The only thing it talks to over the network is BigQuery — and only to query `INFORMATION_SCHEMA.JOBS_BY_PROJECT`.
|
|
6
|
+
|
|
7
|
+
## When to use this vs. the other governor packages
|
|
8
|
+
|
|
9
|
+
- **`governor-audit`** (this package): you have read access to a prod BigQuery project. You want a fast cost audit + detection findings without touching the dbt source code, running dbt, or setting up cloud infrastructure.
|
|
10
|
+
- **`governor-cli`**: you have the dbt project source on your machine and want to run dbt + propose fixes locally.
|
|
11
|
+
- **`governor-web`**: you operate the platform; you want shared infrastructure (GCS-backed manifests, GitHub PRs, scheduled syncs) for a team.
|
|
12
|
+
|
|
13
|
+
## What you get
|
|
14
|
+
|
|
15
|
+
- **Dashboard** — Total / Build / Consumption / Flagged spend KPI cards, top-20 spenders bar chart, paginated cost-drivers table with click-to-sort columns and a per-row issue count.
|
|
16
|
+
- **Detection engine** — every enabled rule from `governor_core.opportunities.rules` (partition_pruning, shuffle_spill, slot_contention, join_explosion, dead_cte, dead_column, dead_window_expression, redundant_order_by, unused_aggregation_output, unused_join) runs on each cached job; opportunities feed the Active Issues pane.
|
|
17
|
+
- **Job detail** — full SQL viewer with expand / copy buttons, metadata panes, and the list of detection-rule findings for that job.
|
|
18
|
+
- **Settings** — Account (gcloud principal + ADC probe), Appearance (light / dark / system), AI / LLM (provider, model, params — reviewer code lands in v0.0.2), Detection Rules (per-rule enable/disable).
|
|
19
|
+
- **Reset cache** — wipe everything `INFORMATION_SCHEMA` collected without losing your config.
|
|
20
|
+
|
|
21
|
+
## Quickstart
|
|
22
|
+
|
|
23
|
+
```sh
|
|
24
|
+
gcloud auth application-default login
|
|
25
|
+
uv tool install governor-audit
|
|
26
|
+
governor-audit init --project prod-warehouse-123 --region us
|
|
27
|
+
governor-audit scan --days 30
|
|
28
|
+
governor-audit start
|
|
29
|
+
# open http://localhost:8765
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The web UI exposes the same actions as the CLI plus the dashboard / settings views. After the first `init` you can do everything from the browser.
|
|
33
|
+
|
|
34
|
+
See [the spec's quickstart](../../specs/141-production-audit/quickstart.md) for the full first-audit walkthrough.
|
|
35
|
+
|
|
36
|
+
## Architecture
|
|
37
|
+
|
|
38
|
+
- **Storage**: SQLite at `~/.governor-audit/state.db` via `governor_core.db.sqlite_compat`.
|
|
39
|
+
- **Auth**: gcloud Application Default Credentials only — `google.auth.default()`. No service-account JSON. No browser OAuth.
|
|
40
|
+
- **Workload classification**: manifest-free heuristic — dbt-originated CTAS / MERGE / INSERT / UPDATE / DELETE → `build`; non-dbt SELECT → `consumption`; ambiguous → `other`. Driven by the `/* {"app": "dbt"` comment-prefix the dbt-bigquery adapter prepends.
|
|
41
|
+
- **Loopback only**: the FastAPI app rejects any request whose `Host:` header isn't a localhost variant. Not a public service.
|
|
42
|
+
|
|
43
|
+
See [spec 141](../../specs/141-production-audit/) for the complete spec set:
|
|
44
|
+
|
|
45
|
+
- [spec.md](../../specs/141-production-audit/spec.md) — feature spec
|
|
46
|
+
- [plan.md](../../specs/141-production-audit/plan.md) — implementation plan
|
|
47
|
+
- [data-model.md](../../specs/141-production-audit/data-model.md) — schemas
|
|
48
|
+
- [contracts/cli.md](../../specs/141-production-audit/contracts/cli.md) — CLI contract
|
|
49
|
+
- [contracts/web-routes.md](../../specs/141-production-audit/contracts/web-routes.md) — web routes contract
|
|
50
|
+
|
|
51
|
+
## Versioning
|
|
52
|
+
|
|
53
|
+
`governor-audit` ships on its own version track, decoupled from the cloud bundle (`governor-core` / `governor-web` / `governor-cli` / `governor-bq`). Audit `v0.0.1` and cloud `v0.7.x` coexist. See [scripts/release-audit.sh](../../scripts/release-audit.sh) for the release flow.
|
|
54
|
+
|
|
55
|
+
## License
|
|
56
|
+
|
|
57
|
+
MIT.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[tool.setuptools.packages.find]
|
|
6
|
+
where = ["src"]
|
|
7
|
+
include = ["governor_audit*"]
|
|
8
|
+
|
|
9
|
+
[tool.setuptools.package-data]
|
|
10
|
+
governor_audit = [
|
|
11
|
+
"web/templates/*.html",
|
|
12
|
+
"web/templates/**/*.html",
|
|
13
|
+
"web/static/**/*",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[project]
|
|
17
|
+
name = "governor-audit"
|
|
18
|
+
version = "0.0.1"
|
|
19
|
+
description = "Read-only BigQuery cost-audit tool — single-user, gcloud ADC only, no GCS / no GitHub / no dbt installation."
|
|
20
|
+
readme = "README.md"
|
|
21
|
+
license = "MIT"
|
|
22
|
+
requires-python = ">=3.14"
|
|
23
|
+
authors = [{name = "Simple Machines", email = "hello@simplemachines.co.nz"}]
|
|
24
|
+
maintainers = [{name = "Simple Machines", email = "hello@simplemachines.co.nz"}]
|
|
25
|
+
keywords = ["bigquery", "cost-optimization", "dbt", "audit", "data-engineering", "warehouse"]
|
|
26
|
+
classifiers = [
|
|
27
|
+
"Development Status :: 4 - Beta",
|
|
28
|
+
"Environment :: Console",
|
|
29
|
+
"Environment :: Web Environment",
|
|
30
|
+
"Intended Audience :: Developers",
|
|
31
|
+
"Intended Audience :: System Administrators",
|
|
32
|
+
"Operating System :: OS Independent",
|
|
33
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
34
|
+
"Programming Language :: Python :: 3.14",
|
|
35
|
+
"Topic :: Database",
|
|
36
|
+
"Topic :: Office/Business :: Financial",
|
|
37
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
38
|
+
"Topic :: System :: Monitoring",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
# governor-core is a sibling package in this monorepo. Until it is
|
|
42
|
+
# published to PyPI, end-user installs from PyPI need a `--find-links`
|
|
43
|
+
# pointing at the GitHub release wheels — see scripts/release-audit.sh.
|
|
44
|
+
# When governor-core lands on PyPI this pin Just Works for `pip install`.
|
|
45
|
+
dependencies = [
|
|
46
|
+
"governor-core==0.7.29",
|
|
47
|
+
"typer>=0.12.0",
|
|
48
|
+
"rich>=13.0.0",
|
|
49
|
+
"fastapi>=0.115.0",
|
|
50
|
+
"jinja2>=3.1.0",
|
|
51
|
+
"uvicorn>=0.30.6",
|
|
52
|
+
"pydantic>=2.0.0",
|
|
53
|
+
"google-cloud-bigquery>=3.20.0",
|
|
54
|
+
"google-auth>=2.30.0",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[project.scripts]
|
|
58
|
+
governor-audit = "governor_audit.cli.app:app"
|
|
59
|
+
|
|
60
|
+
[project.urls]
|
|
61
|
+
Homepage = "https://github.com/simple-machines/governor"
|
|
62
|
+
Repository = "https://github.com/simple-machines/governor"
|
|
63
|
+
Issues = "https://github.com/simple-machines/governor/issues"
|
|
64
|
+
Documentation = "https://github.com/simple-machines/governor/tree/main/specs/141-production-audit"
|
|
65
|
+
Changelog = "https://github.com/simple-machines/governor/releases?q=audit-"
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""governor-audit — read-only production audit tool.
|
|
2
|
+
|
|
3
|
+
Single-user CLI that audits a production dbt deployment using only
|
|
4
|
+
``gcloud`` Application Default Credentials, a user-supplied GCP project
|
|
5
|
+
+ region, and a ``manifest.json`` uploaded from the user's dev
|
|
6
|
+
environment. Queries production BigQuery ``INFORMATION_SCHEMA.JOBS``
|
|
7
|
+
for historical job data, runs the existing ``governor-core`` detection
|
|
8
|
+
rules, and surfaces findings with code-location pointers from the
|
|
9
|
+
manifest.
|
|
10
|
+
|
|
11
|
+
This package is read-only by design: no PR creation, no shadow
|
|
12
|
+
validation, no LLM solution generation in v1. See spec 141 for the
|
|
13
|
+
full posture.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from governor_audit.version import get_app_version
|
|
19
|
+
|
|
20
|
+
__version__ = get_app_version()
|
|
File without changes
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""gcloud Application Default Credentials probe.
|
|
2
|
+
|
|
3
|
+
Single read-only check that proves the user can read INFORMATION_SCHEMA
|
|
4
|
+
in the configured project + region. Returns a typed result so the
|
|
5
|
+
caller (init / doctor / scan) can render the right message.
|
|
6
|
+
|
|
7
|
+
This module never logs the credential bytes — only the principal email
|
|
8
|
+
(``client_email``-equivalent for service accounts; user email for ADC
|
|
9
|
+
sessions) and a sanitised version of any BigQuery error.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from enum import Enum
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ProbeStatus(str, Enum):
|
|
20
|
+
OK = "ok"
|
|
21
|
+
ADC_MISSING = "adc_missing" # No credentials at all (DefaultCredentialsError)
|
|
22
|
+
ADC_INVALID = "adc_invalid" # Credentials present but rejected (RefreshError)
|
|
23
|
+
IAM_INSUFFICIENT = "iam_insufficient" # 403 from BigQuery — needs role grant
|
|
24
|
+
REGION_UNKNOWN = "region_unknown" # 404 region/dataset not found
|
|
25
|
+
UNKNOWN = "unknown" # Some other Google API error — message included verbatim
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class ProbeResult:
|
|
30
|
+
status: ProbeStatus
|
|
31
|
+
principal: str | None # Email of the active gcloud principal, when known.
|
|
32
|
+
message: str # Human-readable, safe to log/print. No credentials.
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def ok(self) -> bool:
|
|
36
|
+
return self.status is ProbeStatus.OK
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
_REQUIRED_IAM_HINT = (
|
|
40
|
+
"Required: roles/bigquery.resourceViewer + roles/bigquery.jobUser "
|
|
41
|
+
"on the project (or a custom role granting bigquery.jobs.list and "
|
|
42
|
+
"bigquery.tables.get on INFORMATION_SCHEMA.SCHEMATA)."
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
_GCLOUD_LOGIN_HINT = "Run: gcloud auth application-default login"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _principal_email_from_credentials(creds: Any) -> str | None:
|
|
49
|
+
"""Try to extract a usable identity from the credentials object.
|
|
50
|
+
Returns ``None`` if we can't (no exception — best-effort)."""
|
|
51
|
+
return (
|
|
52
|
+
getattr(creds, "service_account_email", None)
|
|
53
|
+
or getattr(creds, "_service_account_email", None)
|
|
54
|
+
or getattr(creds, "_target_principal", None)
|
|
55
|
+
or getattr(creds, "_quota_project_id", None)
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def probe_adc(project_id: str, region: str) -> ProbeResult:
|
|
60
|
+
"""Run one read-only INFORMATION_SCHEMA query as a permission check.
|
|
61
|
+
|
|
62
|
+
The query is intentionally minimal:
|
|
63
|
+
|
|
64
|
+
SELECT 1 FROM `<project>.region-<region>`.INFORMATION_SCHEMA.SCHEMATA
|
|
65
|
+
LIMIT 1
|
|
66
|
+
|
|
67
|
+
It's billed as a metadata query (typically free or fractions of a
|
|
68
|
+
cent). Any successful response means the user can read the project
|
|
69
|
+
in the requested region.
|
|
70
|
+
"""
|
|
71
|
+
# Imports are lazy so the module can be imported in environments
|
|
72
|
+
# that don't have google-cloud-bigquery installed (e.g. running the
|
|
73
|
+
# config schema tests in isolation).
|
|
74
|
+
try:
|
|
75
|
+
from google.api_core import exceptions as gcp_exc
|
|
76
|
+
from google.auth import default as google_auth_default
|
|
77
|
+
from google.auth.exceptions import (
|
|
78
|
+
DefaultCredentialsError,
|
|
79
|
+
RefreshError,
|
|
80
|
+
)
|
|
81
|
+
from google.cloud import bigquery
|
|
82
|
+
except ImportError as exc:
|
|
83
|
+
return ProbeResult(
|
|
84
|
+
status=ProbeStatus.UNKNOWN,
|
|
85
|
+
principal=None,
|
|
86
|
+
message=f"google-cloud-bigquery not importable: {exc}",
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# Step 1 — find ADC at all.
|
|
90
|
+
try:
|
|
91
|
+
credentials, _ = google_auth_default(
|
|
92
|
+
scopes=["https://www.googleapis.com/auth/cloud-platform"],
|
|
93
|
+
)
|
|
94
|
+
except DefaultCredentialsError as exc:
|
|
95
|
+
return ProbeResult(
|
|
96
|
+
status=ProbeStatus.ADC_MISSING,
|
|
97
|
+
principal=None,
|
|
98
|
+
message=f"No Application Default Credentials found. {_GCLOUD_LOGIN_HINT}. ({exc})",
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
principal = _principal_email_from_credentials(credentials)
|
|
102
|
+
|
|
103
|
+
# Step 2 — issue the probe query.
|
|
104
|
+
try:
|
|
105
|
+
client = bigquery.Client(project=project_id, credentials=credentials)
|
|
106
|
+
query = (
|
|
107
|
+
f"SELECT 1 FROM `{project_id}.region-{region}`.INFORMATION_SCHEMA.SCHEMATA "
|
|
108
|
+
"LIMIT 1"
|
|
109
|
+
)
|
|
110
|
+
job = client.query(query)
|
|
111
|
+
# Materialise to force the API call; we don't care about the rows.
|
|
112
|
+
list(job.result(max_results=1))
|
|
113
|
+
except RefreshError as exc:
|
|
114
|
+
return ProbeResult(
|
|
115
|
+
status=ProbeStatus.ADC_INVALID,
|
|
116
|
+
principal=principal,
|
|
117
|
+
message=(
|
|
118
|
+
f"Credentials present but rejected by Google. "
|
|
119
|
+
f"{_GCLOUD_LOGIN_HINT}. ({exc})"
|
|
120
|
+
),
|
|
121
|
+
)
|
|
122
|
+
except gcp_exc.Forbidden as exc:
|
|
123
|
+
return ProbeResult(
|
|
124
|
+
status=ProbeStatus.IAM_INSUFFICIENT,
|
|
125
|
+
principal=principal,
|
|
126
|
+
message=(
|
|
127
|
+
f"Permission denied on project {project_id!r}. "
|
|
128
|
+
f"{_REQUIRED_IAM_HINT} ({exc.reason or exc})"
|
|
129
|
+
),
|
|
130
|
+
)
|
|
131
|
+
except gcp_exc.NotFound as exc:
|
|
132
|
+
return ProbeResult(
|
|
133
|
+
status=ProbeStatus.REGION_UNKNOWN,
|
|
134
|
+
principal=principal,
|
|
135
|
+
message=(
|
|
136
|
+
f"Region {region!r} or project {project_id!r} not found by "
|
|
137
|
+
f"BigQuery. Check the region matches a real BigQuery location "
|
|
138
|
+
f"(e.g. 'us', 'australia-southeast1'). ({exc})"
|
|
139
|
+
),
|
|
140
|
+
)
|
|
141
|
+
except gcp_exc.GoogleAPIError as exc:
|
|
142
|
+
return ProbeResult(
|
|
143
|
+
status=ProbeStatus.UNKNOWN,
|
|
144
|
+
principal=principal,
|
|
145
|
+
message=f"BigQuery API error during ADC probe: {exc}",
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
return ProbeResult(
|
|
149
|
+
status=ProbeStatus.OK,
|
|
150
|
+
principal=principal,
|
|
151
|
+
message=(
|
|
152
|
+
f"OK — authenticated{' as ' + principal if principal else ''}, "
|
|
153
|
+
f"INFORMATION_SCHEMA reachable in {project_id}/{region}."
|
|
154
|
+
),
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
__all__ = ["ProbeStatus", "ProbeResult", "probe_adc"]
|
|
File without changes
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Top-level Typer app for the ``governor-audit`` CLI.
|
|
2
|
+
|
|
3
|
+
Exposed via the ``governor-audit`` entry point declared in pyproject.toml.
|
|
4
|
+
Each command lives in its own module under ``cli/commands/`` so the app
|
|
5
|
+
file stays small and the per-command logic can be exercised in isolation.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
from rich.console import Console
|
|
12
|
+
|
|
13
|
+
from governor_audit.cli.commands import init as _init
|
|
14
|
+
from governor_audit.cli.commands import scan as _scan
|
|
15
|
+
from governor_audit.cli.commands import status as _status
|
|
16
|
+
from governor_audit.version import get_app_version
|
|
17
|
+
|
|
18
|
+
app = typer.Typer(
|
|
19
|
+
name="governor-audit",
|
|
20
|
+
no_args_is_help=True,
|
|
21
|
+
add_completion=False,
|
|
22
|
+
help=(
|
|
23
|
+
"Read-only BigQuery + dbt cost-audit tool. Single-user, gcloud "
|
|
24
|
+
"ADC only, no GCS / no GitHub / no dbt. See `governor-audit init --help` "
|
|
25
|
+
"to get started."
|
|
26
|
+
),
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
app.command(name="init")(_init.init_command)
|
|
30
|
+
app.command(name="scan")(_scan.scan_command)
|
|
31
|
+
app.command(name="status")(_status.status_command)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@app.callback(invoke_without_command=True)
|
|
35
|
+
def _root(
|
|
36
|
+
ctx: typer.Context,
|
|
37
|
+
version: bool = typer.Option(
|
|
38
|
+
False,
|
|
39
|
+
"--version",
|
|
40
|
+
"-V",
|
|
41
|
+
help="Print the package version and exit.",
|
|
42
|
+
is_eager=True,
|
|
43
|
+
),
|
|
44
|
+
) -> None:
|
|
45
|
+
if version:
|
|
46
|
+
Console().print(f"governor-audit {get_app_version()}")
|
|
47
|
+
raise typer.Exit(0)
|
|
48
|
+
if ctx.invoked_subcommand is None:
|
|
49
|
+
# No subcommand and no --version → show help and exit 0.
|
|
50
|
+
Console().print(ctx.get_help())
|
|
51
|
+
raise typer.Exit(0)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
if __name__ == "__main__":
|
|
55
|
+
app()
|
|
File without changes
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""``governor-audit init`` — configure the audit target.
|
|
2
|
+
|
|
3
|
+
Per [contracts/cli.md § init](../../../../specs/141-production-audit/contracts/cli.md#init).
|
|
4
|
+
Argument names, exit codes, and message format are part of the
|
|
5
|
+
user-facing contract.
|
|
6
|
+
|
|
7
|
+
**v2 change**: no ``--manifest`` argument. The package never reads a
|
|
8
|
+
manifest. ``--dbt-only-default`` added so the user can persist their
|
|
9
|
+
preferred default for the per-scan ``--dbt-only`` flag.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from typing import Annotated
|
|
15
|
+
|
|
16
|
+
import typer
|
|
17
|
+
from rich.console import Console
|
|
18
|
+
|
|
19
|
+
from governor_audit.auth.adc import ProbeStatus, probe_adc
|
|
20
|
+
from governor_audit.config.loader import (
|
|
21
|
+
ConfigNotFoundError,
|
|
22
|
+
load_config,
|
|
23
|
+
save_config,
|
|
24
|
+
)
|
|
25
|
+
from governor_audit.config.schema import (
|
|
26
|
+
DEFAULT_LOOKBACK_DAYS,
|
|
27
|
+
DEFAULT_MAX_SCAN_BYTES,
|
|
28
|
+
DEFAULT_PORT,
|
|
29
|
+
AuditConfig,
|
|
30
|
+
ScanDefaults,
|
|
31
|
+
WebUIConfig,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
EXIT_OK = 0
|
|
35
|
+
EXIT_GENERIC = 1
|
|
36
|
+
EXIT_BAD_ARGS = 2
|
|
37
|
+
EXIT_ADC_MISSING = 3
|
|
38
|
+
EXIT_CONFIG_CONFLICT = 4
|
|
39
|
+
EXIT_IAM_INSUFFICIENT = 6
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def init_command(
|
|
43
|
+
project: Annotated[
|
|
44
|
+
str,
|
|
45
|
+
typer.Option(
|
|
46
|
+
"--project", help="GCP project ID being audited.", show_default=False
|
|
47
|
+
),
|
|
48
|
+
],
|
|
49
|
+
region: Annotated[
|
|
50
|
+
str,
|
|
51
|
+
typer.Option(
|
|
52
|
+
"--region",
|
|
53
|
+
help="BigQuery region (e.g. 'us', 'australia-southeast1').",
|
|
54
|
+
show_default=False,
|
|
55
|
+
),
|
|
56
|
+
],
|
|
57
|
+
port: Annotated[
|
|
58
|
+
int,
|
|
59
|
+
typer.Option("--port", help="Local web UI port.", min=1024, max=65535),
|
|
60
|
+
] = DEFAULT_PORT,
|
|
61
|
+
max_scan_bytes: Annotated[
|
|
62
|
+
int,
|
|
63
|
+
typer.Option(
|
|
64
|
+
"--max-scan-bytes",
|
|
65
|
+
help="BigQuery dry-run cost cap (bytes-billed). Default 10 GB.",
|
|
66
|
+
min=1_000_000,
|
|
67
|
+
),
|
|
68
|
+
] = DEFAULT_MAX_SCAN_BYTES,
|
|
69
|
+
lookback_days: Annotated[
|
|
70
|
+
int,
|
|
71
|
+
typer.Option(
|
|
72
|
+
"--lookback-days",
|
|
73
|
+
help="Default --days for `scan` when not specified.",
|
|
74
|
+
min=1,
|
|
75
|
+
max=365,
|
|
76
|
+
),
|
|
77
|
+
] = DEFAULT_LOOKBACK_DAYS,
|
|
78
|
+
dbt_only_default: Annotated[
|
|
79
|
+
bool,
|
|
80
|
+
typer.Option(
|
|
81
|
+
"--dbt-only-default/--all-jobs-default",
|
|
82
|
+
help=(
|
|
83
|
+
"Default value of --dbt-only for `scan` when not passed. "
|
|
84
|
+
"When True, scans narrow to queries starting with "
|
|
85
|
+
"/* {\"app\": \"dbt\". Per-scan override always wins."
|
|
86
|
+
),
|
|
87
|
+
),
|
|
88
|
+
] = False,
|
|
89
|
+
force: Annotated[
|
|
90
|
+
bool,
|
|
91
|
+
typer.Option(
|
|
92
|
+
"--force/--no-force",
|
|
93
|
+
help="Overwrite an existing config when fields conflict.",
|
|
94
|
+
),
|
|
95
|
+
] = False,
|
|
96
|
+
) -> None:
|
|
97
|
+
"""Configure governor-audit against a project / region."""
|
|
98
|
+
console = Console()
|
|
99
|
+
|
|
100
|
+
# Step 1 — refuse to silently overwrite a conflicting config.
|
|
101
|
+
if not force:
|
|
102
|
+
try:
|
|
103
|
+
existing = load_config()
|
|
104
|
+
except ConfigNotFoundError:
|
|
105
|
+
existing = None
|
|
106
|
+
if existing is not None:
|
|
107
|
+
conflicts: list[str] = []
|
|
108
|
+
if existing.gcp_project_id != project:
|
|
109
|
+
conflicts.append(
|
|
110
|
+
f" gcp_project_id: {existing.gcp_project_id!r} → {project!r}"
|
|
111
|
+
)
|
|
112
|
+
if existing.bigquery_region.lower() != region.lower():
|
|
113
|
+
conflicts.append(
|
|
114
|
+
f" bigquery_region: {existing.bigquery_region!r} → {region!r}"
|
|
115
|
+
)
|
|
116
|
+
if conflicts:
|
|
117
|
+
console.print(
|
|
118
|
+
"[bold red]Existing config conflicts with the new arguments:[/]\n"
|
|
119
|
+
+ "\n".join(conflicts)
|
|
120
|
+
+ "\n\nRe-run with --force to overwrite, or use "
|
|
121
|
+
"`governor-audit config <key> <value>` for a single field."
|
|
122
|
+
)
|
|
123
|
+
raise typer.Exit(EXIT_CONFIG_CONFLICT)
|
|
124
|
+
|
|
125
|
+
# Step 2 — probe ADC against the requested project + region.
|
|
126
|
+
console.print(f"Probing ADC against {project} / {region}…")
|
|
127
|
+
probe = probe_adc(project, region)
|
|
128
|
+
if probe.status is ProbeStatus.ADC_MISSING:
|
|
129
|
+
console.print(f"[bold red]✗[/] {probe.message}")
|
|
130
|
+
raise typer.Exit(EXIT_ADC_MISSING)
|
|
131
|
+
if probe.status is ProbeStatus.IAM_INSUFFICIENT:
|
|
132
|
+
console.print(f"[bold red]✗[/] {probe.message}")
|
|
133
|
+
raise typer.Exit(EXIT_IAM_INSUFFICIENT)
|
|
134
|
+
if probe.status is ProbeStatus.REGION_UNKNOWN:
|
|
135
|
+
console.print(f"[bold red]✗[/] {probe.message}")
|
|
136
|
+
raise typer.Exit(EXIT_BAD_ARGS)
|
|
137
|
+
if not probe.ok:
|
|
138
|
+
console.print(f"[bold red]✗[/] ADC probe failed: {probe.message}")
|
|
139
|
+
raise typer.Exit(EXIT_GENERIC)
|
|
140
|
+
console.print(
|
|
141
|
+
f"[green]✓[/] ADC verified — authenticated"
|
|
142
|
+
f"{' as ' + probe.principal if probe.principal else ''}"
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
# Step 3 — write config.
|
|
146
|
+
config = AuditConfig.model_validate({
|
|
147
|
+
"gcp_project_id": project,
|
|
148
|
+
"bigquery_region": region,
|
|
149
|
+
"scan_defaults": ScanDefaults(
|
|
150
|
+
lookback_days=lookback_days,
|
|
151
|
+
max_scan_bytes=max_scan_bytes,
|
|
152
|
+
dbt_only=dbt_only_default,
|
|
153
|
+
),
|
|
154
|
+
"web_ui": WebUIConfig(port=port),
|
|
155
|
+
})
|
|
156
|
+
saved_path = save_config(config)
|
|
157
|
+
console.print(f"[green]✓[/] Config written to {saved_path}\n")
|
|
158
|
+
|
|
159
|
+
# Step 4 — friendly hint.
|
|
160
|
+
console.print("Next: run a scan")
|
|
161
|
+
if dbt_only_default:
|
|
162
|
+
console.print(" [cyan]governor-audit scan[/] [dim](dbt-only by default)[/]")
|
|
163
|
+
else:
|
|
164
|
+
console.print(" [cyan]governor-audit scan[/]")
|
|
165
|
+
console.print(" [cyan]governor-audit scan --dbt-only[/] [dim](narrow to dbt-originated jobs)[/]")
|