dataplat 0.1.0__tar.gz

dataplat-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  push:
+    branches: ["**"]
+  pull_request:
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Setup Python
+        run: uv python install 3.13
+
+      - name: Install dependencies
+        run: uv sync --group dev --all-extras
+
+      - name: Ruff
+        run: uv run ruff check .
+
+      - name: MyPy
+        run: uv run mypy dataplat
+
+      - name: Pytest
+        run: uv run pytest

dataplat-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,58 @@
+name: Release
+
+on:
+  push:
+    tags: ["[0-9]*.[0-9]*.[0-9]*"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Build sdist and wheel
+        run: uv build
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # PyPI Trusted Publishing (OIDC)
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    needs: publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Create GitHub release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*
+          generate_release_notes: true

dataplat-0.1.0/.gitignore

@@ -0,0 +1,212 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer,
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+.github/runner
+local/
+
+# Superpowers scratch (plans, specs) — not tracked in this repo.
+docs/superpowers/

dataplat-0.1.0/.python-version

@@ -0,0 +1 @@
+3.13

dataplat-0.1.0/CHANGELOG.md

@@ -0,0 +1,17 @@
+# Changelog
+
+## 0.1.0
+
+Initial public release.
+
+- Optional per-area dependency extras (`dataplat[db,ingest,bi,cloud,all]`)
+  with auto-detection: `dp config sync` installs what your enabled areas
+  need, and hitting a stubbed area offers to install its extra and re-run
+  your command.
+
+- `dp` command with areas per component type: `db`, `ingest` (Airbyte),
+  `bi` (Superset), `cloud` (AWS), `ci` (GitHub runners), plus `status`,
+  `open`, and `config`.
+- Fully config-driven database targets via `DP_TARGETS` and per-target
+  `<NAME>_*` environment variables (Postgres and Redshift engines).
+- `.envrc` loading with a global config link (`dp config init`).

dataplat-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hans Lemm
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dataplat-0.1.0/PKG-INFO

@@ -0,0 +1,326 @@
+Metadata-Version: 2.4
+Name: dataplat
+Version: 0.1.0
+Summary: One command to manage any shape of data platform: databases, ingestion, BI, cloud, and CI.
+Project-URL: Homepage, https://github.com/hanslemm/dataplat
+Project-URL: Repository, https://github.com/hanslemm/dataplat
+Project-URL: Issues, https://github.com/hanslemm/dataplat/issues
+Project-URL: Changelog, https://github.com/hanslemm/dataplat/blob/main/CHANGELOG.md
+Author: Hans Lemm
+License-Expression: MIT
+License-File: LICENSE
+Keywords: airbyte,aws,cli,data-platform,dbt,postgres,redshift,superset
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: System :: Systems Administration
+Classifier: Topic :: Utilities
+Requires-Python: >=3.13
+Requires-Dist: rich>=14.2.0
+Requires-Dist: typer>=0.21.1
+Provides-Extra: all
+Requires-Dist: boto3>=1.35.0; extra == 'all'
+Requires-Dist: croniter>=2.0.0; extra == 'all'
+Requires-Dist: httpx>=0.25.0; extra == 'all'
+Requires-Dist: plotext>=5.3.2; extra == 'all'
+Requires-Dist: psycopg[binary]>=3.2.9; extra == 'all'
+Requires-Dist: textual>=0.64.0; extra == 'all'
+Provides-Extra: bi
+Requires-Dist: httpx>=0.25.0; extra == 'bi'
+Provides-Extra: cloud
+Requires-Dist: boto3>=1.35.0; extra == 'cloud'
+Requires-Dist: plotext>=5.3.2; extra == 'cloud'
+Provides-Extra: db
+Requires-Dist: psycopg[binary]>=3.2.9; extra == 'db'
+Provides-Extra: ingest
+Requires-Dist: croniter>=2.0.0; extra == 'ingest'
+Requires-Dist: httpx>=0.25.0; extra == 'ingest'
+Requires-Dist: textual>=0.64.0; extra == 'ingest'
+Description-Content-Type: text/markdown
+
+# dataplat
+
+**One command to manage any shape of data platform.**
+
+`dataplat` ships a single CLI, `dp`, with an area for each type of component
+in a data platform: warehouses/databases, ingestion, BI, cloud, and CI. Point
+it at *your* stack through environment variables — nothing about your
+infrastructure is hardcoded.
+
+```text
+dp
+├── status                 # one-shot health overview (--json, --no-aws)
+├── open                   # airbyte | superset | rds [id] | redshift | secrets [name]
+├── config                 # init | show | doctor [--connect]
+├── db                     # warehouses & databases (Postgres, Redshift)
+│   ├── query              # ad-hoc SQL (--format table|csv|json, --write guard)
+│   ├── describe           # schema/table/view report (--json)
+│   ├── long-queries       # triage per target (--history, --json)
+│   ├── kill               # cancel/terminate queries by PID
+│   ├── role               # list | show | create | drop
+│   ├── top-tables         # rank big tables (--drop-sql, --drop)
+│   └── dbt-orphans        # scan/rename | revert | purge (--older-than)
+├── ingest                 # data ingestion
+│   └── airbyte
+│       ├── connections    # list | get | create | update | set-cursor
+│       │                  # sync | refresh | reset | delete
+│       ├── jobs           # list | get | cancel
+│       ├── sources        # list | get | create | update | delete
+│       ├── destinations   # list | get | create | update | delete
+│       ├── definitions    # list-sources | list-destinations
+│       ├── workspaces     # list | get
+│       ├── tags           # list | create
+│       └── templates      # source | destination | connection
+├── bi                     # business intelligence
+│   └── superset
+│       ├── users          # list | create | update | delete
+│       ├── roles          # list
+│       └── groups         # list
+├── cloud                  # cloud providers
+│   └── aws
+│       ├── secrets        # list | get | compare | set | edit | rename-key
+│       │                  # describe | versions | rollback | delete | restore
+│       ├── rds            # metrics | plot | list
+│       └── redshift       # metrics
+└── ci                     # build infrastructure
+    └── github
+        └── runner         # start | stop | status
+```
+
+## Installation
+
+```bash
+uv tool install "dataplat[all]"     # recommended: everything
+# or
+pipx install "dataplat[all]"
+# or
+pip install "dataplat[all]"
+```
+
+Each area's dependencies are an optional extra, so you can also install
+only what your platform uses:
+
+| Extra | Enables | Pulls in |
+| --- | --- | --- |
+| `db` | `dp db` | psycopg |
+| `ingest` | `dp ingest` | httpx, textual, croniter |
+| `bi` | `dp bi` | httpx |
+| `cloud` | `dp cloud` | boto3, plotext |
+| `all` | everything | all of the above |
+
+A bare `pip install dataplat` gives you the core (`status`, `open`,
+`config`) with every other area stubbed. You don't have to plan this in
+advance: `dp` knows which areas your configuration enables and installs
+what's missing on demand — see below.
+
+## Auto-installing dependencies
+
+Two ways, both of which detect whether `dp` runs from a uv tool, pipx, or
+plain-venv install and use the matching installer:
+
+```bash
+dp config sync            # detect enabled areas, install missing deps (confirms)
+dp config sync --check    # report only; exit 1 if something is missing (CI-friendly)
+```
+
+Or just use a command: if your config enables an area whose extra is
+missing, `dp db query ...` shows exactly what it will run, asks, installs,
+and re-runs your original command. Non-interactive sessions never install
+silently — they print the command and exit instead. `dp config doctor`
+also reports per-area dependency status.
+
+## Quick start
+
+1. Declare your database targets — any names you like:
+
+   ```bash
+   # ~/.envrc (or any env mechanism you prefer)
+   export DP_TARGETS="warehouse,lake"
+   export DP_DEFAULT_TARGET="warehouse"
+
+   export WAREHOUSE_ENGINE=postgresql
+   export WAREHOUSE_HOST=db.example.com
+   export WAREHOUSE_DATABASE=analytics
+   export WAREHOUSE_USER=me
+   export WAREHOUSE_PASSWORD=…
+
+   export LAKE_ENGINE=redshift
+   export LAKE_HOST=lake.abc123.eu-central-1.redshift-serverless.amazonaws.com
+   export LAKE_DATABASE=dev
+   export LAKE_USER=me
+   export LAKE_PASSWORD=…
+   export LAKE_REASSIGN_OWNER=admin      # role drop reassigns owned objects here
+   ```
+
+2. Optionally link that file globally and check your setup:
+
+   ```bash
+   dp config init --envrc ~/.envrc   # set the global link
+   dp config show                    # which .envrc is active, what's set
+   dp config doctor --connect        # validate config; probe live systems
+   ```
+
+3. Go:
+
+   ```bash
+   dp status
+   dp db query 'SELECT 1'
+   dp db query -t lake 'SELECT 1'
+   ```
+
+## Environment loading
+
+`dp` loads variables from `.envrc` on startup without overriding values
+already set in your shell. Lookup order:
+
+1. `DP_ENVRC_PATH`
+2. `~/.config/dataplat/.envrc` (the global link — set it with `dp config init`)
+3. `.envrc` in the current directory
+
+## Configuration reference
+
+| Variable | Purpose |
+| --- | --- |
+| `DP_TARGETS` | Comma-separated DB target names (e.g. `warehouse,lake`). |
+| `DP_DEFAULT_TARGET` | Target used when `--target` is omitted (default: first of `DP_TARGETS`). |
+| `<NAME>_ENGINE` | `postgresql` (default) or `redshift`, per target. |
+| `<NAME>_HOST/_PORT/_USER/_PASSWORD/_DATABASE/_SSLMODE` | Connection settings, per target. |
+| `<NAME>_REASSIGN_OWNER` | Default owner for `dp db role drop` ownership transfer. |
+| `AIRBYTE_BASE_URL` + `AIRBYTE_CLIENT_ID`/`AIRBYTE_CLIENT_SECRET` (cloud) or `AIRBYTE_EMAIL`/`AIRBYTE_PASSWORD` (OSS) | Airbyte API access. |
+| `SUPERSET_BASE_URL`, `SUPERSET_ADMIN_USERNAME`, `SUPERSET_ADMIN_PASSWORD` | Superset API access. |
+| `DP_AWS_PROFILE` | Default AWS profile for `dp cloud aws` commands. |
+| `DP_AWS_PROFILE_ALIASES` | Short aliases, e.g. `prod=AdminAccess-Prod,qa=AdminAccess-QA`. |
+| `DP_AWS_REGION` | Default AWS region (falls back to `AWS_REGION`, then the profile). |
+| `DP_RDS_INSTANCE` | Default RDS instance for `dp cloud aws rds` / `dp status`. |
+| `DP_DBT_PROJECT` | dbt project name — required for `dp db dbt-orphans`. |
+| `DP_DBT_INVOCATION_COMMAND` | Optional filter on dbt_artifacts invocations. |
+| `DP_DBT_ORPHANS_EXCLUDE_SCHEMAS` | Comma-separated schemas to skip (default `raw,_raw,dbt_artifacts`). |
+| `GHA_APP_ID`, `GHA_APP_PRIVATE_KEY` | GitHub App creds for `dp ci github runner`. |
+| `DP_CI_RUNNER_DNS` | Comma-separated DNS servers for the runner container. |
+
+## Conventions
+
+- **`-t/--target`** — named DB target from `DP_TARGETS`. Multi-target
+  commands accept `all`. Sets the engine and env prefix in one flag;
+  `--engine` / `--env-prefix` remain as overrides.
+- **`--json`** — every read command can emit machine-readable output.
+- **`--yes/-y`** — every destructive or bulk-mutating command confirms first;
+  pass `--yes` in scripts. Bulk mutators also support `--dry-run`.
+- **`--limit/-n`** — row caps share one spelling everywhere.
+- **Secrets stay off argv** — prefer `--value-stdin` / hidden prompts; values
+  are never echoed back.
+
+## Examples
+
+### Daily overview
+
+```bash
+dp status                  # DBs, Airbyte jobs (24h), runners, RDS at a glance
+dp open superset           # jump to a web UI
+```
+
+### DB query
+
+```bash
+dp db query 'SELECT 1'                         # default target
+dp db query -t lake 'SELECT 1'                 # named target
+dp db query --format csv -n 0 'SELECT ...' > out.csv
+echo 'SELECT 1' | dp db query
+dp db query --write 'UPDATE t SET x = 1'       # writes need --write or a confirm
+```
+
+### Long queries and kill
+
+```bash
+dp db long-queries                       # all targets: running + recent failures
+dp db long-queries -t warehouse --history    # pg_stat_statements aggregate
+dp db kill 12345 -t warehouse            # terminate a backend (confirms first)
+```
+
+### DB roles
+
+```bash
+dp db role list --users-only
+dp db role show alice -t lake --json
+dp db role create svc_reporting --table-select reporting --databases analytics
+dp db role create readers --no-login --table-select reporting   # passwordless group role
+dp db role create readers --no-login --grant-to alice,bob
+dp db role drop old_user --all-databases --dry-run
+```
+
+`list`, `create`, and `drop` work against both Postgres and Redshift targets.
+`create` makes login roles with generated passwords by default; `--no-login`
+creates a passwordless group-style role instead. `drop` transfers owned
+objects to the target's `<NAME>_REASSIGN_OWNER` before `DROP USER`.
+
+### Cleanup
+
+```bash
+dp db top-tables --schema-prefix dev_ -n 30
+dp db top-tables --drop-sql > review.sql       # emit a script
+dp db dbt-orphans                              # dry-run scan (default)
+dp db dbt-orphans --no-dry-run                 # apply renames (confirms)
+dp db dbt-orphans purge --older-than 7 --no-dry-run
+dp db dbt-orphans revert                       # undo from the audit log
+```
+
+### AWS secrets
+
+```bash
+dp cloud aws secrets list --prefix /kubernetes
+dp cloud aws secrets get /my/secret --key password
+dp cloud aws secrets compare /my/secret -p prod -p qa
+echo -n "hunter2" | dp cloud aws secrets set my/secret --value-stdin -p qa
+dp cloud aws secrets versions my/secret
+dp cloud aws secrets rollback my/secret        # AWSCURRENT -> AWSPREVIOUS
+```
+
+All writes show their targets and confirm (or `--yes`).
+
+### AWS monitoring
+
+```bash
+dp cloud aws rds metrics --json
+dp cloud aws rds plot -m cpu -m connections --hours 12
+dp cloud aws redshift metrics -w my-workgroup
+```
+
+### Airbyte
+
+```bash
+dp ingest airbyte connections list -w <workspace-id> --json
+dp ingest airbyte connections sync -c <connection-id> --wait
+dp ingest airbyte connections reset -c <connection-id>
+
+# Move every date-based cursor to a date, across all connections from one source
+dp ingest airbyte connections set-cursor --source-id <id> --to 2024-01-01 --dry-run
+dp ingest airbyte connections set-cursor -c <connection-id> --to 2024-01-01 --yes
+
+# xmin (Postgres transaction-id) cursors: set them directly
+dp ingest airbyte connections set-cursor -c <connection-id> --xmin 0 --yes
+```
+
+## Development
+
+```bash
+git clone https://github.com/hanslemm/dataplat
+cd dataplat
+uv sync --group dev --all-extras
+uv run pytest
+uv run ruff check .
+uv run mypy dataplat
+```
+
+## Releasing
+
+Tag `X.Y.Z` (bare semver, no `v` prefix) on `main`; GitHub Actions builds
+and publishes to PyPI via Trusted Publishing. Commits follow
+[Conventional Commits](https://www.conventionalcommits.org/).
+
+## License
+
+[MIT](LICENSE)