bigquery-cleaner 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bigquery_cleaner-0.1.0/.gitignore +64 -0
- bigquery_cleaner-0.1.0/AGENTS.md +81 -0
- bigquery_cleaner-0.1.0/LICENSE +21 -0
- bigquery_cleaner-0.1.0/PKG-INFO +182 -0
- bigquery_cleaner-0.1.0/README.md +167 -0
- bigquery_cleaner-0.1.0/cleaner.example.toml +25 -0
- bigquery_cleaner-0.1.0/deploy.sh +17 -0
- bigquery_cleaner-0.1.0/pyproject.toml +43 -0
- bigquery_cleaner-0.1.0/ruff.toml +35 -0
- bigquery_cleaner-0.1.0/scripts/deploy.py +244 -0
- bigquery_cleaner-0.1.0/src/bigquery_cleaner/__init__.py +10 -0
- bigquery_cleaner-0.1.0/src/bigquery_cleaner/bq_client.py +401 -0
- bigquery_cleaner-0.1.0/src/bigquery_cleaner/cli.py +523 -0
- bigquery_cleaner-0.1.0/src/bigquery_cleaner/config.py +139 -0
- bigquery_cleaner-0.1.0/src/bigquery_cleaner/core_operations.py +315 -0
- bigquery_cleaner-0.1.0/src/bigquery_cleaner/queries/sql.py +66 -0
- bigquery_cleaner-0.1.0/src/bigquery_cleaner/utils.py +91 -0
- bigquery_cleaner-0.1.0/tests/test_cli_imports.py +3 -0
- bigquery_cleaner-0.1.0/uv.lock +692 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
### Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.pyo
|
|
5
|
+
*.pyd
|
|
6
|
+
*.so
|
|
7
|
+
*.dylib
|
|
8
|
+
*.egg
|
|
9
|
+
*.egg-info/
|
|
10
|
+
.eggs/
|
|
11
|
+
MANIFEST
|
|
12
|
+
|
|
13
|
+
### Build artifacts
|
|
14
|
+
build/
|
|
15
|
+
dist/
|
|
16
|
+
pip-wheel-metadata/
|
|
17
|
+
|
|
18
|
+
### Test/coverage
|
|
19
|
+
.pytest_cache/
|
|
20
|
+
.tox/
|
|
21
|
+
.nox/
|
|
22
|
+
.coverage
|
|
23
|
+
.coverage.*
|
|
24
|
+
coverage.xml
|
|
25
|
+
htmlcov/
|
|
26
|
+
|
|
27
|
+
### Type checkers / linters
|
|
28
|
+
.mypy_cache/
|
|
29
|
+
.pytype/
|
|
30
|
+
.pyre/
|
|
31
|
+
.ruff_cache/
|
|
32
|
+
|
|
33
|
+
### Environments
|
|
34
|
+
.venv/
|
|
35
|
+
venv/
|
|
36
|
+
ENV/
|
|
37
|
+
env/
|
|
38
|
+
.env
|
|
39
|
+
.env.*
|
|
40
|
+
.conda/
|
|
41
|
+
|
|
42
|
+
### Editors/OS
|
|
43
|
+
.idea/
|
|
44
|
+
*.iml
|
|
45
|
+
.vscode/
|
|
46
|
+
*.code-workspace
|
|
47
|
+
.DS_Store
|
|
48
|
+
Thumbs.db
|
|
49
|
+
*.swp
|
|
50
|
+
*.swo
|
|
51
|
+
|
|
52
|
+
### Jupyter
|
|
53
|
+
.ipynb_checkpoints/
|
|
54
|
+
.jupyter/
|
|
55
|
+
|
|
56
|
+
### Logs
|
|
57
|
+
*.log
|
|
58
|
+
|
|
59
|
+
### uv
|
|
60
|
+
.uv/
|
|
61
|
+
|
|
62
|
+
.junie/
|
|
63
|
+
|
|
64
|
+
.private/
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# BigQuery Cleaner — Agent Guide
|
|
2
|
+
|
|
3
|
+
This repository is a uv‑managed Python package exposing a Typer CLI to help identify (and later, rename/delete) unused BigQuery tables. Use this document to quickly align with project decisions and conventions before making changes.
|
|
4
|
+
|
|
5
|
+
## Project Shape
|
|
6
|
+
- Packaging: `src/` layout
|
|
7
|
+
- Manager: `uv` (Python 3.10+)
|
|
8
|
+
- CLI: Typer entrypoint `bigquery-cleaner`
|
|
9
|
+
- GCP: `google-cloud-bigquery` client with ADC auth
|
|
10
|
+
- Console script: defined in `pyproject.toml` as `bigquery-cleaner = "bigquery_cleaner.cli:app"`
|
|
11
|
+
|
|
12
|
+
## Core Commands
|
|
13
|
+
- `list-unused-tables`
|
|
14
|
+
- Purpose: list tables not referenced by queries in the past N days AND modified more than N days ago.
|
|
15
|
+
- Inputs: provided via TOML config and/or CLI flags (flags override config).
|
|
16
|
+
- No `--dataset` flag. Prefer `--datasets` (comma‑separated) or `--all-datasets`.
|
|
17
|
+
- Output: displays table ID, creation/modification dates, and size in GB. Includes per-dataset totals and a grand total (table count and size).
|
|
18
|
+
- `rename-old-tables`
|
|
19
|
+
- Purpose: rename tables not referenced by queries in the past N days AND modified more than N days ago.
|
|
20
|
+
- Suffix: uses `--suffix` or `rename_suffix` from config.
|
|
21
|
+
- `revert-renamed-tables`
|
|
22
|
+
- Purpose: revert renamed tables by removing the specified suffix.
|
|
23
|
+
- Suffix: uses `--suffix` or `rename_suffix` from config.
|
|
24
|
+
|
|
25
|
+
## Config Schema (TOML)
|
|
26
|
+
Section: `[bigquery_cleaner]`
|
|
27
|
+
- `project` (str): GCP project id
|
|
28
|
+
- `datasets` (list[str], optional): dataset ids; may be fully‑qualified (`proj.ds`) or just `ds`
|
|
29
|
+
- `exclude_datasets` (list[str], optional): dataset ids to skip
|
|
30
|
+
- `all_datasets` (bool, optional): scan every dataset in `project`
|
|
31
|
+
- `days` (int, default 30): lookback window
|
|
32
|
+
- `location` (str, optional): generally auto‑detected per dataset; not required
|
|
33
|
+
- `rename_suffix` (str, default "_renamed_YYYYMMDD"): suffix for `rename-old-tables`
|
|
34
|
+
- `dry_run` (bool, default false): if true, do not perform actual modifications
|
|
35
|
+
- `log_level` (str, default "INFO"): logging level
|
|
36
|
+
|
|
37
|
+
Example: see `cleaner.example.toml`.
|
|
38
|
+
|
|
39
|
+
## Implementation Notes
|
|
40
|
+
- Detection functions live in `src/bigquery_cleaner/core_operations.py`.
|
|
41
|
+
- Orchestration and utility helpers live in `src/bigquery_cleaner/utils.py`.
|
|
42
|
+
- API and client-related functions live in `src/bigquery_cleaner/bq_client.py`.
|
|
43
|
+
- CLI commands use `Annotated` types for parameters to avoid Ruff B008 errors (no function calls in argument defaults).
|
|
44
|
+
- Core detection functions:
|
|
45
|
+
- `get_non_queried_tables(cfg: CleanerConfig) -> dict[str, list[TableMetadata]]`
|
|
46
|
+
- `get_old_tables(cfg: CleanerConfig) -> dict[str, list[TableMetadata]]`
|
|
47
|
+
- `get_old_modified_tables(cfg: CleanerConfig) -> dict[str, list[TableMetadata]]`
|
|
48
|
+
- Strategy:
|
|
49
|
+
- Query `{project}.region-<location>.INFORMATION_SCHEMA.JOBS` and use `referenced_tables` to find recently referenced tables (per dataset, per location).
|
|
50
|
+
- Query `INFORMATION_SCHEMA.TABLE_STORAGE` for table size and metadata.
|
|
51
|
+
- List all tables in the dataset via the BigQuery client and subtract the referenced set.
|
|
52
|
+
- Intersection: used to find tables that are both unqueried AND haven't been modified in N days.
|
|
53
|
+
- Queries run in the dataset’s location; location is auto‑resolved from dataset metadata when not provided.
|
|
54
|
+
- CLI wiring in `src/bigquery_cleaner/cli.py` respects precedence: CLI flags > TOML config.
|
|
55
|
+
- Do not add repeatable flags (Click `multiple=True`); use comma‑separated `--datasets` instead.
|
|
56
|
+
|
|
57
|
+
## Constraints & Conventions
|
|
58
|
+
- Keep the `src/` layout. Align with existing style and minimal changes.
|
|
59
|
+
- Use `Annotated` for all Typer options and arguments.
|
|
60
|
+
- The file `bigquery_maintenance.py` is a legacy reference; do not modify it. It will be replaced as CLI matures.
|
|
61
|
+
- Prefer config‑first UX; only add flags that add real value. Avoid introducing `--dataset` (single) again.
|
|
62
|
+
- Windows usage: use `uv run bigquery-cleaner ...` or install via `uv tool install .` to get `bigquery-cleaner` on PATH.
|
|
63
|
+
- Linting: Use Ruff for linting and formatting. Configuration is in `ruff.toml`.
|
|
64
|
+
- Run check: `uv run ruff check .`
|
|
65
|
+
- Run format: `uv run ruff format .`
|
|
66
|
+
|
|
67
|
+
## Auth & Prereqs
|
|
68
|
+
- ADC expected (e.g., `gcloud auth application-default login`) or `GOOGLE_APPLICATION_CREDENTIALS` set.
|
|
69
|
+
|
|
70
|
+
## Roadmap (future work)
|
|
71
|
+
- Add `delete-old` command with `--dry-run`.
|
|
72
|
+
- Dependency graph checks: ensure no views/materialized views/procedures reference candidate tables (e.g., `INFORMATION_SCHEMA.OBJECT_REFERENCES`).
|
|
73
|
+
- Audit logs usage: consider Cloud Logging signals (dataRead/jobCompleted) for non-SQL consumers (extracts, copies, ML, BI tools).
|
|
74
|
+
- Scheduled jobs: detect Scheduled Queries/Dataform/Composer/Dataflow that read tables (via jobs metadata or config sources).
|
|
75
|
+
- Table-type handling: treat views/materialized views/external tables/snapshots carefully; avoid deleting sources.
|
|
76
|
+
- Streaming/loads: skip tables with active streaming buffers or very recent loads.
|
|
77
|
+
- Partitions: consider per-partition recency; avoid deleting tables with recent partitions.
|
|
78
|
+
- Labels/tags/policies: honor governance flags (e.g., `do-not-delete`, policy tags, constraints).
|
|
79
|
+
- Expiration: skip tables with upcoming expiry and leverage TTL where possible.
|
|
80
|
+
- Safety workflow: dry-run report, owner approval, optional snapshot/copy backup before DROP.
|
|
81
|
+
- Improve logging vs print, and structured output options (e.g., `--json`).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Alan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bigquery-cleaner
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CLI tool to clean up your BigQuery old and unused datasets and tables.
|
|
5
|
+
Author: Alan Vainsencher
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Requires-Dist: google-auth>=2.0
|
|
10
|
+
Requires-Dist: google-cloud-bigquery>=3.25
|
|
11
|
+
Requires-Dist: rich>=14.2.0
|
|
12
|
+
Requires-Dist: tomli>=2.0; python_version < '3.11'
|
|
13
|
+
Requires-Dist: typer>=0.12
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# 🧹 BigQuery Cleaner
|
|
17
|
+
|
|
18
|
+
[](https://www.python.org/downloads/)
|
|
19
|
+
[](https://github.com/astral-sh/uv)
|
|
20
|
+
[](https://opensource.org/licenses/MIT)
|
|
21
|
+
|
|
22
|
+
**BigQuery Cleaner** is a powerful CLI tool designed to help you declutter your Google BigQuery environment. It identifies tables that haven't been queried recently and provides safe mechanisms to rename or prepare them for deletion.
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## 🚀 Quick Start
|
|
27
|
+
|
|
28
|
+
Get up and running in seconds:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
# 1. Install via uv
|
|
32
|
+
uv tool install .
|
|
33
|
+
|
|
34
|
+
# 2. Find unused tables (older than 30 days and not queried)
|
|
35
|
+
bigquery-cleaner list-unused-tables --project your-gcp-project --all-datasets --days 30
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## ✨ Features
|
|
41
|
+
|
|
42
|
+
- 🔍 **Unused Table Detection**: Scans `INFORMATION_SCHEMA.JOBS` to find tables that aren't being used.
|
|
43
|
+
- 📊 **Storage Insight**: Displays table sizes in GB and provides per-dataset and grand total summaries.
|
|
44
|
+
- 📂 **Multi-Dataset Support**: Target specific datasets, exclude others, or scan your entire project.
|
|
45
|
+
- 🏷️ **Safe Renaming**: Dry-run mode allows you to see what *would* happen before making changes.
|
|
46
|
+
- 🔄 **Easy Revert**: Renamed a table by mistake? Revert it easily with the `revert-renamed-tables` command.
|
|
47
|
+
- 🗑️ **Permanent Cleanup**: Use `delete-tables` to remove suffixed tables once you've confirmed they are no longer needed.
|
|
48
|
+
- 🧹 **Dataset Cleanup**: Remove empty datasets that no longer contain any tables or views using `delete-empty-datasets`.
|
|
49
|
+
- ⚙️ **Configurable**: Use a `cleaner.toml` file to save your project defaults and lookback windows.
|
|
50
|
+
- ⚡ **Built with Speed**: Powered by `uv`, `Typer`, and `Rich` for a beautiful, fast terminal experience.
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## 📋 Prerequisites
|
|
55
|
+
|
|
56
|
+
- **Python 3.10+**
|
|
57
|
+
- **[uv](https://docs.astral.sh/uv/)** package manager installed.
|
|
58
|
+
- **Google Cloud Credentials**: Configured via Application Default Credentials (ADC).
|
|
59
|
+
```bash
|
|
60
|
+
gcloud auth application-default login
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
## 🛠️ Installation
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
# Clone the repository
|
|
69
|
+
git clone https://github.com/your-repo/bigquery-cleaner.git
|
|
70
|
+
cd bigquery-cleaner
|
|
71
|
+
|
|
72
|
+
# Sync dependencies and install the tool
|
|
73
|
+
uv sync
|
|
74
|
+
uv tool install .
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## 📖 Usage Guide
|
|
80
|
+
|
|
81
|
+
### Help Command
|
|
82
|
+
> Every command and sub-command supports the `--help` flag for detailed information on available options.
|
|
83
|
+
>
|
|
84
|
+
>Example: `bigquery-cleaner list-unused-tables --help`
|
|
85
|
+
>
|
|
86
|
+
> Run bigquery-cleaner --help to see all available commands.
|
|
87
|
+
|
|
88
|
+
### Connectivity Check
|
|
89
|
+
Ensure your credentials and project access are working:
|
|
90
|
+
```bash
|
|
91
|
+
bigquery-cleaner ping --project YOUR_PROJECT
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### Exploration
|
|
95
|
+
List available datasets and tables:
|
|
96
|
+
```bash
|
|
97
|
+
# List all datasets
|
|
98
|
+
bigquery-cleaner datasets --project YOUR_PROJECT
|
|
99
|
+
|
|
100
|
+
# List tables in specific datasets
|
|
101
|
+
bigquery-cleaner tables --datasets dataset1,dataset2 --project YOUR_PROJECT
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Identifying Waste
|
|
105
|
+
The core functionality to find old, unreferenced tables:
|
|
106
|
+
```bash
|
|
107
|
+
# List unused tables across all datasets
|
|
108
|
+
bigquery-cleaner list-unused-tables --all-datasets --days 90
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Cleanup Operations
|
|
112
|
+
Safely rename unused tables with a suffix:
|
|
113
|
+
```bash
|
|
114
|
+
# Dry run first!
|
|
115
|
+
bigquery-cleaner rename-old-tables --all-datasets --days 90 --dry-run
|
|
116
|
+
|
|
117
|
+
# Perform the rename
|
|
118
|
+
bigquery-cleaner rename-old-tables --all-datasets --days 90
|
|
119
|
+
|
|
120
|
+
# Delete renamed tables after verification
|
|
121
|
+
# Dry run first!
|
|
122
|
+
bigquery-cleaner delete-tables --all-datasets --suffix "_renamed_20241225" --dry-run
|
|
123
|
+
|
|
124
|
+
# Perform the deletion
|
|
125
|
+
bigquery-cleaner delete-tables --all-datasets --suffix "_renamed_20241225"
|
|
126
|
+
|
|
127
|
+
# Remove empty datasets
|
|
128
|
+
bigquery-cleaner delete-empty-datasets --all-datasets
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
## ⚙️ Configuration
|
|
134
|
+
|
|
135
|
+
Tired of typing the same flags? Create a `cleaner.toml` file in your project root. All CLI options can be persisted here:
|
|
136
|
+
|
|
137
|
+
```toml
|
|
138
|
+
[bigquery_cleaner]
|
|
139
|
+
# GCP Project ID (defaults to ADC project if omitted)
|
|
140
|
+
project = "your-gcp-project"
|
|
141
|
+
|
|
142
|
+
# List of datasets to scan (e.g. ["ds1", "project2.ds2"])
|
|
143
|
+
datasets = ["dataset1", "dataset2"]
|
|
144
|
+
|
|
145
|
+
# List of datasets to ignore
|
|
146
|
+
exclude_datasets = ["logs_dataset", "temp_staging"]
|
|
147
|
+
|
|
148
|
+
# If true, scans all datasets in the project (overrides 'datasets' list)
|
|
149
|
+
all_datasets = true
|
|
150
|
+
|
|
151
|
+
# Lookback window in days for identifying unused tables (default: 30)
|
|
152
|
+
days = 60
|
|
153
|
+
|
|
154
|
+
# Suffix used for renaming and identifying tables for deletion (default: _renamed_YYYYMMDD)
|
|
155
|
+
rename_suffix = "_old_backup"
|
|
156
|
+
|
|
157
|
+
# Default behavior for commands (true = dry run by default)
|
|
158
|
+
dry_run = false
|
|
159
|
+
|
|
160
|
+
# Logging level (DEBUG, INFO, WARNING, ERROR)
|
|
161
|
+
log_level = "INFO"
|
|
162
|
+
|
|
163
|
+
# BigQuery Location (e.g. "US", "EU").
|
|
164
|
+
# Note: Multi-dataset mode usually auto-detects this.
|
|
165
|
+
location = "US"
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Then run with:
|
|
169
|
+
```bash
|
|
170
|
+
bigquery-cleaner list-unused-tables --config cleaner.toml
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
---
|
|
174
|
+
|
|
175
|
+
## 📝 Notes
|
|
176
|
+
|
|
177
|
+
- **Detection Logic**: The `list-unused-tables` command identifies tables created more than `N` days ago that do not appear in `INFORMATION_SCHEMA.JOBS.referenced_tables` within that same window.
|
|
178
|
+
- **Rich Output**: All results are displayed in beautiful, sortable tables thanks to the `Rich` library. Includes total table counts and storage size summaries.
|
|
179
|
+
- **Linting & Quality**: The project uses **Ruff** for fast linting and formatting.
|
|
180
|
+
|
|
181
|
+
---
|
|
182
|
+
Developed by Alan Vainsencher.
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# 🧹 BigQuery Cleaner
|
|
2
|
+
|
|
3
|
+
[](https://www.python.org/downloads/)
|
|
4
|
+
[](https://github.com/astral-sh/uv)
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
|
|
7
|
+
**BigQuery Cleaner** is a powerful CLI tool designed to help you declutter your Google BigQuery environment. It identifies tables that haven't been queried recently and provides safe mechanisms to rename or prepare them for deletion.
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## 🚀 Quick Start
|
|
12
|
+
|
|
13
|
+
Get up and running in seconds:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
# 1. Install via uv
|
|
17
|
+
uv tool install .
|
|
18
|
+
|
|
19
|
+
# 2. Find unused tables (older than 30 days and not queried)
|
|
20
|
+
bigquery-cleaner list-unused-tables --project your-gcp-project --all-datasets --days 30
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## ✨ Features
|
|
26
|
+
|
|
27
|
+
- 🔍 **Unused Table Detection**: Scans `INFORMATION_SCHEMA.JOBS` to find tables that aren't being used.
|
|
28
|
+
- 📊 **Storage Insight**: Displays table sizes in GB and provides per-dataset and grand total summaries.
|
|
29
|
+
- 📂 **Multi-Dataset Support**: Target specific datasets, exclude others, or scan your entire project.
|
|
30
|
+
- 🏷️ **Safe Renaming**: Dry-run mode allows you to see what *would* happen before making changes.
|
|
31
|
+
- 🔄 **Easy Revert**: Renamed a table by mistake? Revert it easily with the `revert-renamed-tables` command.
|
|
32
|
+
- 🗑️ **Permanent Cleanup**: Use `delete-tables` to remove suffixed tables once you've confirmed they are no longer needed.
|
|
33
|
+
- 🧹 **Dataset Cleanup**: Remove empty datasets that no longer contain any tables or views using `delete-empty-datasets`.
|
|
34
|
+
- ⚙️ **Configurable**: Use a `cleaner.toml` file to save your project defaults and lookback windows.
|
|
35
|
+
- ⚡ **Built with Speed**: Powered by `uv`, `Typer`, and `Rich` for a beautiful, fast terminal experience.
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
## 📋 Prerequisites
|
|
40
|
+
|
|
41
|
+
- **Python 3.10+**
|
|
42
|
+
- **[uv](https://docs.astral.sh/uv/)** package manager installed.
|
|
43
|
+
- **Google Cloud Credentials**: Configured via Application Default Credentials (ADC).
|
|
44
|
+
```bash
|
|
45
|
+
gcloud auth application-default login
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## 🛠️ Installation
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
# Clone the repository
|
|
54
|
+
git clone https://github.com/your-repo/bigquery-cleaner.git
|
|
55
|
+
cd bigquery-cleaner
|
|
56
|
+
|
|
57
|
+
# Sync dependencies and install the tool
|
|
58
|
+
uv sync
|
|
59
|
+
uv tool install .
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## 📖 Usage Guide
|
|
65
|
+
|
|
66
|
+
### Help Command
|
|
67
|
+
> Every command and sub-command supports the `--help` flag for detailed information on available options.
|
|
68
|
+
>
|
|
69
|
+
>Example: `bigquery-cleaner list-unused-tables --help`
|
|
70
|
+
>
|
|
71
|
+
> Run bigquery-cleaner --help to see all available commands.
|
|
72
|
+
|
|
73
|
+
### Connectivity Check
|
|
74
|
+
Ensure your credentials and project access are working:
|
|
75
|
+
```bash
|
|
76
|
+
bigquery-cleaner ping --project YOUR_PROJECT
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Exploration
|
|
80
|
+
List available datasets and tables:
|
|
81
|
+
```bash
|
|
82
|
+
# List all datasets
|
|
83
|
+
bigquery-cleaner datasets --project YOUR_PROJECT
|
|
84
|
+
|
|
85
|
+
# List tables in specific datasets
|
|
86
|
+
bigquery-cleaner tables --datasets dataset1,dataset2 --project YOUR_PROJECT
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### Identifying Waste
|
|
90
|
+
The core functionality to find old, unreferenced tables:
|
|
91
|
+
```bash
|
|
92
|
+
# List unused tables across all datasets
|
|
93
|
+
bigquery-cleaner list-unused-tables --all-datasets --days 90
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Cleanup Operations
|
|
97
|
+
Safely rename unused tables with a suffix:
|
|
98
|
+
```bash
|
|
99
|
+
# Dry run first!
|
|
100
|
+
bigquery-cleaner rename-old-tables --all-datasets --days 90 --dry-run
|
|
101
|
+
|
|
102
|
+
# Perform the rename
|
|
103
|
+
bigquery-cleaner rename-old-tables --all-datasets --days 90
|
|
104
|
+
|
|
105
|
+
# Delete renamed tables after verification
|
|
106
|
+
# Dry run first!
|
|
107
|
+
bigquery-cleaner delete-tables --all-datasets --suffix "_renamed_20241225" --dry-run
|
|
108
|
+
|
|
109
|
+
# Perform the deletion
|
|
110
|
+
bigquery-cleaner delete-tables --all-datasets --suffix "_renamed_20241225"
|
|
111
|
+
|
|
112
|
+
# Remove empty datasets
|
|
113
|
+
bigquery-cleaner delete-empty-datasets --all-datasets
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
## ⚙️ Configuration
|
|
119
|
+
|
|
120
|
+
Tired of typing the same flags? Create a `cleaner.toml` file in your project root. All CLI options can be persisted here:
|
|
121
|
+
|
|
122
|
+
```toml
|
|
123
|
+
[bigquery_cleaner]
|
|
124
|
+
# GCP Project ID (defaults to ADC project if omitted)
|
|
125
|
+
project = "your-gcp-project"
|
|
126
|
+
|
|
127
|
+
# List of datasets to scan (e.g. ["ds1", "project2.ds2"])
|
|
128
|
+
datasets = ["dataset1", "dataset2"]
|
|
129
|
+
|
|
130
|
+
# List of datasets to ignore
|
|
131
|
+
exclude_datasets = ["logs_dataset", "temp_staging"]
|
|
132
|
+
|
|
133
|
+
# If true, scans all datasets in the project (overrides 'datasets' list)
|
|
134
|
+
all_datasets = true
|
|
135
|
+
|
|
136
|
+
# Lookback window in days for identifying unused tables (default: 30)
|
|
137
|
+
days = 60
|
|
138
|
+
|
|
139
|
+
# Suffix used for renaming and identifying tables for deletion (default: _renamed_YYYYMMDD)
|
|
140
|
+
rename_suffix = "_old_backup"
|
|
141
|
+
|
|
142
|
+
# Default behavior for commands (true = dry run by default)
|
|
143
|
+
dry_run = false
|
|
144
|
+
|
|
145
|
+
# Logging level (DEBUG, INFO, WARNING, ERROR)
|
|
146
|
+
log_level = "INFO"
|
|
147
|
+
|
|
148
|
+
# BigQuery Location (e.g. "US", "EU").
|
|
149
|
+
# Note: Multi-dataset mode usually auto-detects this.
|
|
150
|
+
location = "US"
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Then run with:
|
|
154
|
+
```bash
|
|
155
|
+
bigquery-cleaner list-unused-tables --config cleaner.toml
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## 📝 Notes
|
|
161
|
+
|
|
162
|
+
- **Detection Logic**: The `list-unused-tables` command identifies tables created more than `N` days ago that do not appear in `INFORMATION_SCHEMA.JOBS.referenced_tables` within that same window.
|
|
163
|
+
- **Rich Output**: All results are displayed in beautiful, sortable tables thanks to the `Rich` library. Includes total table counts and storage size summaries.
|
|
164
|
+
- **Linting & Quality**: The project uses **Ruff** for fast linting and formatting.
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
Developed by Alan Vainsencher.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Sample configuration for bigquery-cleaner
|
|
2
|
+
# Copy to cleaner.toml and edit values, or pass this file directly with --config
|
|
3
|
+
|
|
4
|
+
[bigquery_cleaner]
|
|
5
|
+
# GCP project ID (e.g., "my-project")
|
|
6
|
+
project = "YOUR_PROJECT"
|
|
7
|
+
|
|
8
|
+
# Datasets to inspect; set one or many. Elements can be fully-qualified or rely on project above.
|
|
9
|
+
# datasets = ["your_dataset", "another_ds", "proj.other_dataset"]
|
|
10
|
+
|
|
11
|
+
# Datasets to skip during analysis.
|
|
12
|
+
# exclude_datasets = ["test_dataset", "legacy_ds"]
|
|
13
|
+
|
|
14
|
+
# Scan all datasets in the project (alternative to the above).
|
|
15
|
+
# all_datasets = true
|
|
16
|
+
|
|
17
|
+
# Lookback window in days to consider a table as "unused" (not referenced by queries)
|
|
18
|
+
days = 120
|
|
19
|
+
|
|
20
|
+
# Suffix used when renaming tables (e.g., table_name -> table_name_renamed_20241225)
|
|
21
|
+
# rename_suffix = "_renamed_20241225"
|
|
22
|
+
# dry_run = false
|
|
23
|
+
|
|
24
|
+
# Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL). Defaults to INFO.
|
|
25
|
+
# log_level = "INFO"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
# Thin wrapper that delegates to the Python deploy script.
|
|
5
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
6
|
+
|
|
7
|
+
# Prefer python3, then python; on Windows Git Bash, fall back to 'py -3'.
|
|
8
|
+
if command -v python3 >/dev/null 2>&1; then
|
|
9
|
+
exec python3 "$SCRIPT_DIR/scripts/deploy.py" "$@"
|
|
10
|
+
elif command -v python >/dev/null 2>&1; then
|
|
11
|
+
exec python "$SCRIPT_DIR/scripts/deploy.py" "$@"
|
|
12
|
+
elif command -v py >/dev/null 2>&1; then
|
|
13
|
+
exec py -3 "$SCRIPT_DIR/scripts/deploy.py" "$@"
|
|
14
|
+
else
|
|
15
|
+
echo "Python 3 not found. Install Python 3 and ensure 'python3' or 'python' or 'py' is on PATH." >&2
|
|
16
|
+
exit 1
|
|
17
|
+
fi
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "bigquery-cleaner"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "CLI tool to clean up your BigQuery old and unused datasets and tables."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "Alan Vainsencher" }
|
|
10
|
+
]
|
|
11
|
+
dependencies = [
|
|
12
|
+
"typer>=0.12",
|
|
13
|
+
"google-cloud-bigquery>=3.25",
|
|
14
|
+
"google-auth>=2.0",
|
|
15
|
+
"tomli>=2.0; python_version < '3.11'",
|
|
16
|
+
"rich>=14.2.0",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.scripts]
|
|
20
|
+
bigquery-cleaner = "bigquery_cleaner.cli:app"
|
|
21
|
+
|
|
22
|
+
[tool.uv]
|
|
23
|
+
dev-dependencies = [
|
|
24
|
+
"ruff>=0.6.9",
|
|
25
|
+
"pytest>=8.3",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[build-system]
|
|
29
|
+
requires = ["hatchling>=1.26"]
|
|
30
|
+
build-backend = "hatchling.build"
|
|
31
|
+
|
|
32
|
+
[tool.hatch.build.targets.wheel]
|
|
33
|
+
packages = ["src/bigquery_cleaner"]
|
|
34
|
+
|
|
35
|
+
[tool.hatch.build]
|
|
36
|
+
artifacts = []
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
[[tool.uv.index]]
|
|
40
|
+
name = "testpypi"
|
|
41
|
+
url = "https://test.pypi.org/simple/"
|
|
42
|
+
publish-url = "https://test.pypi.org/legacy/"
|
|
43
|
+
explicit = true
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Ruff configuration file
|
|
2
|
+
# See https://docs.astral.sh/ruff/settings/ for all settings
|
|
3
|
+
|
|
4
|
+
line-length = 100
|
|
5
|
+
target-version = "py310"
|
|
6
|
+
|
|
7
|
+
[lint]
|
|
8
|
+
# Enable Pyflakes (`F`), pycodestyle (`E`, `W`), isort (`I`),
|
|
9
|
+
# pep8-naming (`N`), and various other common rules.
|
|
10
|
+
select = [
|
|
11
|
+
"F", # Pyflakes
|
|
12
|
+
"E", # pycodestyle errors
|
|
13
|
+
"W", # pycodestyle warnings
|
|
14
|
+
"I", # isort
|
|
15
|
+
"N", # pep8-naming
|
|
16
|
+
"UP", # pyupgrade
|
|
17
|
+
"B", # flake8-bugbear
|
|
18
|
+
"C4", # flake8-comprehensions
|
|
19
|
+
"SIM", # flake8-simplify
|
|
20
|
+
"RUF", # Ruff-specific rules
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
# List of rule codes to ignore.
|
|
24
|
+
ignore = [
|
|
25
|
+
"E501", # Line too long (already handled by formatter)
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[lint.isort]
|
|
29
|
+
known-first-party = ["bigquery_cleaner"]
|
|
30
|
+
|
|
31
|
+
[format]
|
|
32
|
+
quote-style = "double"
|
|
33
|
+
indent-style = "space"
|
|
34
|
+
skip-magic-trailing-comma = false
|
|
35
|
+
line-ending = "auto"
|