bigquery-cleaner 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,64 @@
1
+ ### Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.pyo
5
+ *.pyd
6
+ *.so
7
+ *.dylib
8
+ *.egg
9
+ *.egg-info/
10
+ .eggs/
11
+ MANIFEST
12
+
13
+ ### Build artifacts
14
+ build/
15
+ dist/
16
+ pip-wheel-metadata/
17
+
18
+ ### Test/coverage
19
+ .pytest_cache/
20
+ .tox/
21
+ .nox/
22
+ .coverage
23
+ .coverage.*
24
+ coverage.xml
25
+ htmlcov/
26
+
27
+ ### Type checkers / linters
28
+ .mypy_cache/
29
+ .pytype/
30
+ .pyre/
31
+ .ruff_cache/
32
+
33
+ ### Environments
34
+ .venv/
35
+ venv/
36
+ ENV/
37
+ env/
38
+ .env
39
+ .env.*
40
+ .conda/
41
+
42
+ ### Editors/OS
43
+ .idea/
44
+ *.iml
45
+ .vscode/
46
+ *.code-workspace
47
+ .DS_Store
48
+ Thumbs.db
49
+ *.swp
50
+ *.swo
51
+
52
+ ### Jupyter
53
+ .ipynb_checkpoints/
54
+ .jupyter/
55
+
56
+ ### Logs
57
+ *.log
58
+
59
+ ### uv
60
+ .uv/
61
+
62
+ .junie/
63
+
64
+ .private/
@@ -0,0 +1,81 @@
1
+ # BigQuery Cleaner — Agent Guide
2
+
3
+ This repository is a uv‑managed Python package exposing a Typer CLI to help identify (and later, rename/delete) unused BigQuery tables. Use this document to quickly align with project decisions and conventions before making changes.
4
+
5
+ ## Project Shape
6
+ - Packaging: `src/` layout
7
+ - Manager: `uv` (Python 3.10+)
8
+ - CLI: Typer entrypoint `bigquery-cleaner`
9
+ - GCP: `google-cloud-bigquery` client with ADC auth
10
+ - Console script: defined in `pyproject.toml` as `bigquery-cleaner = "bigquery_cleaner.cli:app"`
11
+
12
+ ## Core Commands
13
+ - `list-unused-tables`
14
+ - Purpose: list tables not referenced by queries in the past N days AND modified more than N days ago.
15
+ - Inputs: provided via TOML config and/or CLI flags (flags override config).
16
+ - No `--dataset` flag. Prefer `--datasets` (comma‑separated) or `--all-datasets`.
17
+ - Output: displays table ID, creation/modification dates, and size in GB. Includes per-dataset totals and a grand total (table count and size).
18
+ - `rename-old-tables`
19
+ - Purpose: rename tables not referenced by queries in the past N days AND modified more than N days ago.
20
+ - Suffix: uses `--suffix` or `rename_suffix` from config.
21
+ - `revert-renamed-tables`
22
+ - Purpose: revert renamed tables by removing the specified suffix.
23
+ - Suffix: uses `--suffix` or `rename_suffix` from config.
24
+
25
+ ## Config Schema (TOML)
26
+ Section: `[bigquery_cleaner]`
27
+ - `project` (str): GCP project id
28
+ - `datasets` (list[str], optional): dataset ids; may be fully‑qualified (`proj.ds`) or just `ds`
29
+ - `exclude_datasets` (list[str], optional): dataset ids to skip
30
+ - `all_datasets` (bool, optional): scan every dataset in `project`
31
+ - `days` (int, default 30): lookback window
32
+ - `location` (str, optional): generally auto‑detected per dataset; not required
33
+ - `rename_suffix` (str, default "_renamed_YYYYMMDD"): suffix for `rename-old-tables`
34
+ - `dry_run` (bool, default false): if true, do not perform actual modifications
35
+ - `log_level` (str, default "INFO"): logging level
36
+
37
+ Example: see `cleaner.example.toml`.
38
+
39
+ ## Implementation Notes
40
+ - Detection functions live in `src/bigquery_cleaner/core_operations.py`.
41
+ - Orchestration and utility helpers live in `src/bigquery_cleaner/utils.py`.
42
+ - API and client-related functions live in `src/bigquery_cleaner/bq_client.py`.
43
+ - CLI commands use `Annotated` types for parameters to avoid Ruff B008 errors (no function calls in argument defaults).
44
+ - Core detection functions:
45
+ - `get_non_queried_tables(cfg: CleanerConfig) -> dict[str, list[TableMetadata]]`
46
+ - `get_old_tables(cfg: CleanerConfig) -> dict[str, list[TableMetadata]]`
47
+ - `get_old_modified_tables(cfg: CleanerConfig) -> dict[str, list[TableMetadata]]`
48
+ - Strategy:
49
+ - Query `{project}.region-<location>.INFORMATION_SCHEMA.JOBS` and use `referenced_tables` to find recently referenced tables (per dataset, per location).
50
+ - Query `INFORMATION_SCHEMA.TABLE_STORAGE` for table size and metadata.
51
+ - List all tables in the dataset via the BigQuery client and subtract the referenced set.
52
+ - Intersection: used to find tables that are both unqueried AND haven't been modified in N days.
53
+ - Queries run in the dataset’s location; location is auto‑resolved from dataset metadata when not provided.
54
+ - CLI wiring in `src/bigquery_cleaner/cli.py` respects precedence: CLI flags > TOML config.
55
+ - Do not add repeatable flags (Click `multiple=True`); use comma‑separated `--datasets` instead.
56
+
57
+ ## Constraints & Conventions
58
+ - Keep the `src/` layout. Align with existing style and minimal changes.
59
+ - Use `Annotated` for all Typer options and arguments.
60
+ - The file `bigquery_maintenance.py` is a legacy reference; do not modify it. It will be replaced as CLI matures.
61
+ - Prefer config‑first UX; only add flags that add real value. Avoid introducing `--dataset` (single) again.
62
+ - Windows usage: use `uv run bigquery-cleaner ...` or install via `uv tool install .` to get `bigquery-cleaner` on PATH.
63
+ - Linting: Use Ruff for linting and formatting. Configuration is in `ruff.toml`.
64
+ - Run check: `uv run ruff check .`
65
+ - Run format: `uv run ruff format .`
66
+
67
+ ## Auth & Prereqs
68
+ - ADC expected (e.g., `gcloud auth application-default login`) or `GOOGLE_APPLICATION_CREDENTIALS` set.
69
+
70
+ ## Roadmap (future work)
71
+ - Add `delete-old` command with `--dry-run`.
72
+ - Dependency graph checks: ensure no views/materialized views/procedures reference candidate tables (e.g., `INFORMATION_SCHEMA.OBJECT_REFERENCES`).
73
+ - Audit logs usage: consider Cloud Logging signals (dataRead/jobCompleted) for non-SQL consumers (extracts, copies, ML, BI tools).
74
+ - Scheduled jobs: detect Scheduled Queries/Dataform/Composer/Dataflow that read tables (via jobs metadata or config sources).
75
+ - Table-type handling: treat views/materialized views/external tables/snapshots carefully; avoid deleting sources.
76
+ - Streaming/loads: skip tables with active streaming buffers or very recent loads.
77
+ - Partitions: consider per-partition recency; avoid deleting tables with recent partitions.
78
+ - Labels/tags/policies: honor governance flags (e.g., `do-not-delete`, policy tags, constraints).
79
+ - Expiration: skip tables with upcoming expiry and leverage TTL where possible.
80
+ - Safety workflow: dry-run report, owner approval, optional snapshot/copy backup before DROP.
81
+ - Improve logging vs print, and structured output options (e.g., `--json`).
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Alan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,182 @@
1
+ Metadata-Version: 2.4
2
+ Name: bigquery-cleaner
3
+ Version: 0.1.0
4
+ Summary: CLI tool to clean up your BigQuery old and unused datasets and tables.
5
+ Author: Alan Vainsencher
6
+ License-Expression: MIT
7
+ License-File: LICENSE
8
+ Requires-Python: >=3.10
9
+ Requires-Dist: google-auth>=2.0
10
+ Requires-Dist: google-cloud-bigquery>=3.25
11
+ Requires-Dist: rich>=14.2.0
12
+ Requires-Dist: tomli>=2.0; python_version < '3.11'
13
+ Requires-Dist: typer>=0.12
14
+ Description-Content-Type: text/markdown
15
+
16
+ # 🧹 BigQuery Cleaner
17
+
18
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
19
+ [![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
20
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
21
+
22
+ **BigQuery Cleaner** is a powerful CLI tool designed to help you declutter your Google BigQuery environment. It identifies tables that haven't been queried recently and provides safe mechanisms to rename or prepare them for deletion.
23
+
24
+ ---
25
+
26
+ ## 🚀 Quick Start
27
+
28
+ Get up and running in seconds:
29
+
30
+ ```bash
31
+ # 1. Install via uv
32
+ uv tool install .
33
+
34
+ # 2. Find unused tables (older than 30 days and not queried)
35
+ bigquery-cleaner list-unused-tables --project your-gcp-project --all-datasets --days 30
36
+ ```
37
+
38
+ ---
39
+
40
+ ## ✨ Features
41
+
42
+ - 🔍 **Unused Table Detection**: Scans `INFORMATION_SCHEMA.JOBS` to find tables that aren't being used.
43
+ - 📊 **Storage Insight**: Displays table sizes in GB and provides per-dataset and grand total summaries.
44
+ - 📂 **Multi-Dataset Support**: Target specific datasets, exclude others, or scan your entire project.
45
+ - 🏷️ **Safe Renaming**: Dry-run mode allows you to see what *would* happen before making changes.
46
+ - 🔄 **Easy Revert**: Renamed a table by mistake? Revert it easily with the `revert-renamed-tables` command.
47
+ - 🗑️ **Permanent Cleanup**: Use `delete-tables` to remove suffixed tables once you've confirmed they are no longer needed.
48
+ - 🧹 **Dataset Cleanup**: Remove empty datasets that no longer contain any tables or views using `delete-empty-datasets`.
49
+ - ⚙️ **Configurable**: Use a `cleaner.toml` file to save your project defaults and lookback windows.
50
+ - ⚡ **Built with Speed**: Powered by `uv`, `Typer`, and `Rich` for a beautiful, fast terminal experience.
51
+
52
+ ---
53
+
54
+ ## 📋 Prerequisites
55
+
56
+ - **Python 3.10+**
57
+ - **[uv](https://docs.astral.sh/uv/)** package manager installed.
58
+ - **Google Cloud Credentials**: Configured via Application Default Credentials (ADC).
59
+ ```bash
60
+ gcloud auth application-default login
61
+ ```
62
+
63
+ ---
64
+
65
+ ## 🛠️ Installation
66
+
67
+ ```bash
68
+ # Clone the repository
69
+ git clone https://github.com/your-repo/bigquery-cleaner.git
70
+ cd bigquery-cleaner
71
+
72
+ # Sync dependencies and install the tool
73
+ uv sync
74
+ uv tool install .
75
+ ```
76
+
77
+ ---
78
+
79
+ ## 📖 Usage Guide
80
+
81
+ ### Help Command
82
+ > Every command and sub-command supports the `--help` flag for detailed information on available options.
83
+ >
84
+ >Example: `bigquery-cleaner list-unused-tables --help`
85
+ >
86
+ > Run bigquery-cleaner --help to see all available commands.
87
+
88
+ ### Connectivity Check
89
+ Ensure your credentials and project access are working:
90
+ ```bash
91
+ bigquery-cleaner ping --project YOUR_PROJECT
92
+ ```
93
+
94
+ ### Exploration
95
+ List available datasets and tables:
96
+ ```bash
97
+ # List all datasets
98
+ bigquery-cleaner datasets --project YOUR_PROJECT
99
+
100
+ # List tables in specific datasets
101
+ bigquery-cleaner tables --datasets dataset1,dataset2 --project YOUR_PROJECT
102
+ ```
103
+
104
+ ### Identifying Waste
105
+ The core functionality to find old, unreferenced tables:
106
+ ```bash
107
+ # List unused tables across all datasets
108
+ bigquery-cleaner list-unused-tables --all-datasets --days 90
109
+ ```
110
+
111
+ ### Cleanup Operations
112
+ Safely rename unused tables with a suffix:
113
+ ```bash
114
+ # Dry run first!
115
+ bigquery-cleaner rename-old-tables --all-datasets --days 90 --dry-run
116
+
117
+ # Perform the rename
118
+ bigquery-cleaner rename-old-tables --all-datasets --days 90
119
+
120
+ # Delete renamed tables after verification
121
+ # Dry run first!
122
+ bigquery-cleaner delete-tables --all-datasets --suffix "_renamed_20241225" --dry-run
123
+
124
+ # Perform the deletion
125
+ bigquery-cleaner delete-tables --all-datasets --suffix "_renamed_20241225"
126
+
127
+ # Remove empty datasets
128
+ bigquery-cleaner delete-empty-datasets --all-datasets
129
+ ```
130
+
131
+ ---
132
+
133
+ ## ⚙️ Configuration
134
+
135
+ Tired of typing the same flags? Create a `cleaner.toml` file in your project root. All CLI options can be persisted here:
136
+
137
+ ```toml
138
+ [bigquery_cleaner]
139
+ # GCP Project ID (defaults to ADC project if omitted)
140
+ project = "your-gcp-project"
141
+
142
+ # List of datasets to scan (e.g. ["ds1", "project2.ds2"])
143
+ datasets = ["dataset1", "dataset2"]
144
+
145
+ # List of datasets to ignore
146
+ exclude_datasets = ["logs_dataset", "temp_staging"]
147
+
148
+ # If true, scans all datasets in the project (overrides 'datasets' list)
149
+ all_datasets = true
150
+
151
+ # Lookback window in days for identifying unused tables (default: 30)
152
+ days = 60
153
+
154
+ # Suffix used for renaming and identifying tables for deletion (default: _renamed_YYYYMMDD)
155
+ rename_suffix = "_old_backup"
156
+
157
+ # Default behavior for commands (true = dry run by default)
158
+ dry_run = false
159
+
160
+ # Logging level (DEBUG, INFO, WARNING, ERROR)
161
+ log_level = "INFO"
162
+
163
+ # BigQuery Location (e.g. "US", "EU").
164
+ # Note: Multi-dataset mode usually auto-detects this.
165
+ location = "US"
166
+ ```
167
+
168
+ Then run with:
169
+ ```bash
170
+ bigquery-cleaner list-unused-tables --config cleaner.toml
171
+ ```
172
+
173
+ ---
174
+
175
+ ## 📝 Notes
176
+
177
+ - **Detection Logic**: The `list-unused-tables` command identifies tables created more than `N` days ago that do not appear in `INFORMATION_SCHEMA.JOBS.referenced_tables` within that same window.
178
+ - **Rich Output**: All results are displayed in beautiful, sortable tables thanks to the `Rich` library. Includes total table counts and storage size summaries.
179
+ - **Linting & Quality**: The project uses **Ruff** for fast linting and formatting.
180
+
181
+ ---
182
+ Developed by Alan Vainsencher.
@@ -0,0 +1,167 @@
1
+ # 🧹 BigQuery Cleaner
2
+
3
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
4
+ [![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
6
+
7
+ **BigQuery Cleaner** is a powerful CLI tool designed to help you declutter your Google BigQuery environment. It identifies tables that haven't been queried recently and provides safe mechanisms to rename or prepare them for deletion.
8
+
9
+ ---
10
+
11
+ ## 🚀 Quick Start
12
+
13
+ Get up and running in seconds:
14
+
15
+ ```bash
16
+ # 1. Install via uv
17
+ uv tool install .
18
+
19
+ # 2. Find unused tables (older than 30 days and not queried)
20
+ bigquery-cleaner list-unused-tables --project your-gcp-project --all-datasets --days 30
21
+ ```
22
+
23
+ ---
24
+
25
+ ## ✨ Features
26
+
27
+ - 🔍 **Unused Table Detection**: Scans `INFORMATION_SCHEMA.JOBS` to find tables that aren't being used.
28
+ - 📊 **Storage Insight**: Displays table sizes in GB and provides per-dataset and grand total summaries.
29
+ - 📂 **Multi-Dataset Support**: Target specific datasets, exclude others, or scan your entire project.
30
+ - 🏷️ **Safe Renaming**: Dry-run mode allows you to see what *would* happen before making changes.
31
+ - 🔄 **Easy Revert**: Renamed a table by mistake? Revert it easily with the `revert-renamed-tables` command.
32
+ - 🗑️ **Permanent Cleanup**: Use `delete-tables` to remove suffixed tables once you've confirmed they are no longer needed.
33
+ - 🧹 **Dataset Cleanup**: Remove empty datasets that no longer contain any tables or views using `delete-empty-datasets`.
34
+ - ⚙️ **Configurable**: Use a `cleaner.toml` file to save your project defaults and lookback windows.
35
+ - ⚡ **Built with Speed**: Powered by `uv`, `Typer`, and `Rich` for a beautiful, fast terminal experience.
36
+
37
+ ---
38
+
39
+ ## 📋 Prerequisites
40
+
41
+ - **Python 3.10+**
42
+ - **[uv](https://docs.astral.sh/uv/)** package manager installed.
43
+ - **Google Cloud Credentials**: Configured via Application Default Credentials (ADC).
44
+ ```bash
45
+ gcloud auth application-default login
46
+ ```
47
+
48
+ ---
49
+
50
+ ## 🛠️ Installation
51
+
52
+ ```bash
53
+ # Clone the repository
54
+ git clone https://github.com/your-repo/bigquery-cleaner.git
55
+ cd bigquery-cleaner
56
+
57
+ # Sync dependencies and install the tool
58
+ uv sync
59
+ uv tool install .
60
+ ```
61
+
62
+ ---
63
+
64
+ ## 📖 Usage Guide
65
+
66
+ ### Help Command
67
+ > Every command and sub-command supports the `--help` flag for detailed information on available options.
68
+ >
69
+ >Example: `bigquery-cleaner list-unused-tables --help`
70
+ >
71
+ > Run bigquery-cleaner --help to see all available commands.
72
+
73
+ ### Connectivity Check
74
+ Ensure your credentials and project access are working:
75
+ ```bash
76
+ bigquery-cleaner ping --project YOUR_PROJECT
77
+ ```
78
+
79
+ ### Exploration
80
+ List available datasets and tables:
81
+ ```bash
82
+ # List all datasets
83
+ bigquery-cleaner datasets --project YOUR_PROJECT
84
+
85
+ # List tables in specific datasets
86
+ bigquery-cleaner tables --datasets dataset1,dataset2 --project YOUR_PROJECT
87
+ ```
88
+
89
+ ### Identifying Waste
90
+ The core functionality to find old, unreferenced tables:
91
+ ```bash
92
+ # List unused tables across all datasets
93
+ bigquery-cleaner list-unused-tables --all-datasets --days 90
94
+ ```
95
+
96
+ ### Cleanup Operations
97
+ Safely rename unused tables with a suffix:
98
+ ```bash
99
+ # Dry run first!
100
+ bigquery-cleaner rename-old-tables --all-datasets --days 90 --dry-run
101
+
102
+ # Perform the rename
103
+ bigquery-cleaner rename-old-tables --all-datasets --days 90
104
+
105
+ # Delete renamed tables after verification
106
+ # Dry run first!
107
+ bigquery-cleaner delete-tables --all-datasets --suffix "_renamed_20241225" --dry-run
108
+
109
+ # Perform the deletion
110
+ bigquery-cleaner delete-tables --all-datasets --suffix "_renamed_20241225"
111
+
112
+ # Remove empty datasets
113
+ bigquery-cleaner delete-empty-datasets --all-datasets
114
+ ```
115
+
116
+ ---
117
+
118
+ ## ⚙️ Configuration
119
+
120
+ Tired of typing the same flags? Create a `cleaner.toml` file in your project root. All CLI options can be persisted here:
121
+
122
+ ```toml
123
+ [bigquery_cleaner]
124
+ # GCP Project ID (defaults to ADC project if omitted)
125
+ project = "your-gcp-project"
126
+
127
+ # List of datasets to scan (e.g. ["ds1", "project2.ds2"])
128
+ datasets = ["dataset1", "dataset2"]
129
+
130
+ # List of datasets to ignore
131
+ exclude_datasets = ["logs_dataset", "temp_staging"]
132
+
133
+ # If true, scans all datasets in the project (overrides 'datasets' list)
134
+ all_datasets = true
135
+
136
+ # Lookback window in days for identifying unused tables (default: 30)
137
+ days = 60
138
+
139
+ # Suffix used for renaming and identifying tables for deletion (default: _renamed_YYYYMMDD)
140
+ rename_suffix = "_old_backup"
141
+
142
+ # Default behavior for commands (true = dry run by default)
143
+ dry_run = false
144
+
145
+ # Logging level (DEBUG, INFO, WARNING, ERROR)
146
+ log_level = "INFO"
147
+
148
+ # BigQuery Location (e.g. "US", "EU").
149
+ # Note: Multi-dataset mode usually auto-detects this.
150
+ location = "US"
151
+ ```
152
+
153
+ Then run with:
154
+ ```bash
155
+ bigquery-cleaner list-unused-tables --config cleaner.toml
156
+ ```
157
+
158
+ ---
159
+
160
+ ## 📝 Notes
161
+
162
+ - **Detection Logic**: The `list-unused-tables` command identifies tables created more than `N` days ago that do not appear in `INFORMATION_SCHEMA.JOBS.referenced_tables` within that same window.
163
+ - **Rich Output**: All results are displayed in beautiful, sortable tables thanks to the `Rich` library. Includes total table counts and storage size summaries.
164
+ - **Linting & Quality**: The project uses **Ruff** for fast linting and formatting.
165
+
166
+ ---
167
+ Developed by Alan Vainsencher.
@@ -0,0 +1,25 @@
1
+ # Sample configuration for bigquery-cleaner
2
+ # Copy to cleaner.toml and edit values, or pass this file directly with --config
3
+
4
+ [bigquery_cleaner]
5
+ # GCP project ID (e.g., "my-project")
6
+ project = "YOUR_PROJECT"
7
+
8
+ # Datasets to inspect; set one or many. Elements can be fully-qualified or rely on project above.
9
+ # datasets = ["your_dataset", "another_ds", "proj.other_dataset"]
10
+
11
+ # Datasets to skip during analysis.
12
+ # exclude_datasets = ["test_dataset", "legacy_ds"]
13
+
14
+ # Scan all datasets in the project (alternative to the above).
15
+ # all_datasets = true
16
+
17
+ # Lookback window in days to consider a table as "unused" (not referenced by queries)
18
+ days = 120
19
+
20
+ # Suffix used when renaming tables (e.g., table_name -> table_name_renamed_20241225)
21
+ # rename_suffix = "_renamed_20241225"
22
+ # dry_run = false
23
+
24
+ # Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL). Defaults to INFO.
25
+ # log_level = "INFO"
@@ -0,0 +1,17 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ # Thin wrapper that delegates to the Python deploy script.
5
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
6
+
7
+ # Prefer python3, then python; on Windows Git Bash, fall back to 'py -3'.
8
+ if command -v python3 >/dev/null 2>&1; then
9
+ exec python3 "$SCRIPT_DIR/scripts/deploy.py" "$@"
10
+ elif command -v python >/dev/null 2>&1; then
11
+ exec python "$SCRIPT_DIR/scripts/deploy.py" "$@"
12
+ elif command -v py >/dev/null 2>&1; then
13
+ exec py -3 "$SCRIPT_DIR/scripts/deploy.py" "$@"
14
+ else
15
+ echo "Python 3 not found. Install Python 3 and ensure 'python3' or 'python' or 'py' is on PATH." >&2
16
+ exit 1
17
+ fi
@@ -0,0 +1,43 @@
1
+ [project]
2
+ name = "bigquery-cleaner"
3
+ version = "0.1.0"
4
+ description = "CLI tool to clean up your BigQuery old and unused datasets and tables."
5
+ readme = "README.md"
6
+ requires-python = ">=3.10"
7
+ license = "MIT"
8
+ authors = [
9
+ { name = "Alan Vainsencher" }
10
+ ]
11
+ dependencies = [
12
+ "typer>=0.12",
13
+ "google-cloud-bigquery>=3.25",
14
+ "google-auth>=2.0",
15
+ "tomli>=2.0; python_version < '3.11'",
16
+ "rich>=14.2.0",
17
+ ]
18
+
19
+ [project.scripts]
20
+ bigquery-cleaner = "bigquery_cleaner.cli:app"
21
+
22
+ [tool.uv]
23
+ dev-dependencies = [
24
+ "ruff>=0.6.9",
25
+ "pytest>=8.3",
26
+ ]
27
+
28
+ [build-system]
29
+ requires = ["hatchling>=1.26"]
30
+ build-backend = "hatchling.build"
31
+
32
+ [tool.hatch.build.targets.wheel]
33
+ packages = ["src/bigquery_cleaner"]
34
+
35
+ [tool.hatch.build]
36
+ artifacts = []
37
+
38
+
39
+ [[tool.uv.index]]
40
+ name = "testpypi"
41
+ url = "https://test.pypi.org/simple/"
42
+ publish-url = "https://test.pypi.org/legacy/"
43
+ explicit = true
@@ -0,0 +1,35 @@
1
+ # Ruff configuration file
2
+ # See https://docs.astral.sh/ruff/settings/ for all settings
3
+
4
+ line-length = 100
5
+ target-version = "py310"
6
+
7
+ [lint]
8
+ # Enable Pyflakes (`F`), pycodestyle (`E`, `W`), isort (`I`),
9
+ # pep8-naming (`N`), and various other common rules.
10
+ select = [
11
+ "F", # Pyflakes
12
+ "E", # pycodestyle errors
13
+ "W", # pycodestyle warnings
14
+ "I", # isort
15
+ "N", # pep8-naming
16
+ "UP", # pyupgrade
17
+ "B", # flake8-bugbear
18
+ "C4", # flake8-comprehensions
19
+ "SIM", # flake8-simplify
20
+ "RUF", # Ruff-specific rules
21
+ ]
22
+
23
+ # List of rule codes to ignore.
24
+ ignore = [
25
+ "E501", # Line too long (already handled by formatter)
26
+ ]
27
+
28
+ [lint.isort]
29
+ known-first-party = ["bigquery_cleaner"]
30
+
31
+ [format]
32
+ quote-style = "double"
33
+ indent-style = "space"
34
+ skip-magic-trailing-comma = false
35
+ line-ending = "auto"