dataplat 0.1.0__tar.gz → 0.2.0__tar.gz

dataplat-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,48 @@
+name: CI
+
+on:
+  push:
+    branches: ["**"]
+    # A tag push matches no branch, so without this a release tag would run no
+    # checks at all before release.yml publishes it.
+    tags: ["[0-9]*.[0-9]*.[0-9]*"]
+  pull_request:
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    strategy:
+      # The floor and the ceiling break for different reasons; report both
+      # rather than hiding one behind the other.
+      fail-fast: false
+      matrix:
+        # Lowest version requires-python allows, plus the dev pin.
+        python-version: ["3.12", "3.13"]
+    env:
+      # Overrides .python-version (3.13). Without it every uv command in the
+      # 3.12 leg would still pick the pin and we would test 3.13 twice.
+      UV_PYTHON: ${{ matrix.python-version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Setup Python
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --group dev --all-extras
+
+      - name: Ruff
+        run: uv run ruff check .
+
+      - name: Ruff format
+        run: uv run ruff format --check .
+
+      - name: MyPy
+        run: uv run mypy dataplat
+
+      - name: Pytest
+        run: uv run pytest

dataplat-0.2.0/.github/workflows/release.yml

@@ -0,0 +1,119 @@
+name: Release
+
+on:
+  push:
+    tags: ["[0-9]*.[0-9]*.[0-9]*"]
+
+jobs:
+  # Nothing downstream re-checks the tree, so this job is the only thing
+  # standing between an untested commit and PyPI. Publishing is irreversible:
+  # a version can be yanked but never replaced.
+  verify:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # Must mirror ci.yml: the wheel advertises >=3.12, so the floor has to
+        # pass before we publish, not just the dev pin.
+        python-version: ["3.12", "3.13"]
+    env:
+      UV_PYTHON: ${{ matrix.python-version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Setup Python
+        run: uv python install ${{ matrix.python-version }}
+
+      # Before the expensive sync: a mismatch means the tag would either
+      # publish the wrong version number or die at upload against an already
+      # occupied filename.
+      - name: Assert tag matches project version
+        run: |
+          uv run --no-project python - <<'PY'
+          import os
+          import sys
+          import tomllib
+
+          tag = os.environ["GITHUB_REF_NAME"]
+          with open("pyproject.toml", "rb") as handle:
+              version = tomllib.load(handle)["project"]["version"]
+          if tag != version:
+              sys.exit(
+                  f"Tag '{tag}' does not match [project].version '{version}' in "
+                  "pyproject.toml. Bump the version, commit, and re-tag; "
+                  "refusing to publish a mismatched artifact."
+              )
+          print(f"Tag '{tag}' matches [project].version.")
+          PY
+
+      - name: Install dependencies
+        run: uv sync --group dev --all-extras
+
+      - name: Ruff
+        run: uv run ruff check .
+
+      - name: Ruff format
+        run: uv run ruff format --check .
+
+      - name: MyPy
+        run: uv run mypy dataplat
+
+      - name: Pytest
+        run: uv run pytest
+
+  build:
+    needs: verify
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Build sdist and wheel
+        run: uv build
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # PyPI Trusted Publishing (OIDC)
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    needs: publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Create GitHub release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*
+          generate_release_notes: true

dataplat-0.2.0/CHANGELOG.md

@@ -0,0 +1,123 @@
+# Changelog
+
+## 0.2.0
+
+### Fixed
+
+- Data containing square brackets no longer breaks output. Rich reads `[...]`
+  as markup in every string it renders, so a warehouse value, identifier,
+  comment, secret, API payload, or driver message containing `[/x]` aborted the
+  command with a `MarkupError` traceback, and one containing a style name like
+  `[bold]` was silently swallowed — the output quietly misreported the data.
+  Every renderer now emits external values as literal text, table *headers*
+  included: `dp ingest airbyte connections list --all-columns` takes its column
+  labels straight from Airbyte's JSON keys, and the Textual TUI passes the same
+  labels to a `DataTable`, so both had the identical failure.
+
+- `dp db dbt-orphans revert` and `purge --older-than` could not find their own
+  audit logs if the log directory's path contained a bracket — a home directory
+  named `[work]` was enough. The discovery glob interpolated the directory
+  without escaping it, so the brackets read as a character class and matched
+  nothing: `revert` reported no history, and `purge --older-than` treated every
+  object as having no recorded rename.
+
+- Installing a missing extra no longer changes anything else about your
+  install. It was resolved unpinned, so hitting a stubbed area could upgrade
+  `dataplat` itself and then re-exec your command against the new code; and
+  because `--force` rebuilds the environment from the spec it is given,
+  installing `db` silently removed an `ingest` you already had. The command is
+  now pinned to the running version and unions the extras already present.
+
+- `dp ci github runner start|stop|status` identified containers with an
+  unanchored `docker --filter name=`, which is a substring match: a runner
+  named `foo` matched `gha-runner-foobar`, so `status` could report a
+  different runner's state as yours. All filters are anchored.
+
+- `dp status --json` emitted valid JSON only while nothing needed saying. An
+  expired SSO token wrote its notice to stdout, mixed into the document.
+  Spinners and notices now go to stderr; stdout carries only the payload.
+
+- Pressing TAB inside an area whose extra is missing misbehaved badly. Shell
+  completion walks the command tree with click's resilient parsing, which calls
+  the same resolution hook the missing-extra stub overrode, so completion ran
+  the interactive installer: the offer text landed on the stream the shell
+  evaluates (zsh even re-invoked `dp` from the backticks in the message), a
+  full traceback went to the terminal, and under bash the install itself
+  executed from a keypress. Completion now falls through silently.
+
+- Installing an extra inside a virtualenv that has no `pip` — the default for
+  `uv venv` — prescribed `python -m pip install` and could only fail, reporting
+  just a bare exit code. It now uses `uv pip install --python <env>` when pip is
+  absent, and says so plainly when neither installer is reachable.
+
+- The `Will run:` line is shell-quoted. It prints a command the user may want
+  to run themselves, but the requirement spec's brackets (and now its `==`
+  pin) meant pasting it into zsh failed on globbing.
+
+### Added
+
+- `DP_ENVRC_ALLOW_CWD=0` stops `.envrc` being picked up from the current
+  directory. Loading it made every command sensitive to where you stood, with
+  no way to see that had happened: `dp config show` and `dp config doctor` now
+  name the active file and which candidate produced it, and warn when it came
+  from the working directory.
+
+- `DP_AWS_PROFILE_ALIASES` now works across the whole `dp cloud aws` group.
+  Only `secrets` resolved aliases, so `dp cloud aws rds metrics -p prod` passed
+  the literal `prod` to boto3 and failed with a profile-not-found error while
+  the same flag worked one command over. `rds list` and `redshift metrics` also
+  print the profile they actually used rather than the shorthand you typed.
+
+- Python 3.12 support. `requires-python` was 3.13 while the only construct
+  above 3.11 in the package is one PEP 695 generic, so 3.12 users hit an
+  install failure for nothing. CI now tests the advertised floor as well as
+  the dev pin.
+
+- The release workflow verifies that the pushed tag matches
+  `[project].version` and runs the full check matrix before building. CI also
+  runs on tag pushes — a tag matched no branch filter, so releases were
+  published without any checks having run.
+
+### Changed
+
+- Areas mount lazily, cutting `dp --version` from 360 ms to 100 ms. Every
+  area's dependencies — psycopg, textual, httpx, boto3 — were imported at
+  startup regardless of the command; now nothing heavy loads until the area
+  that needs it is actually used. Areas resolve through the registry
+  (`dataplat/core/registry.py`) and carry their own dependency contract, so a
+  future third-party area is no longer tied to a lookup in a built-in dict.
+
+- One confirmation gate covers every destructive command. Three idioms had
+  drifted apart and only one told you what to do when stdin was not a TTY;
+  non-interactive runs now consistently name the flag that would have worked
+  and exit 1 rather than aborting with no explanation.
+
+  **Behaviour change worth knowing before you upgrade:** piping a confirmation
+  into a destructive command no longer authorizes it. `echo y | dp db role drop
+  old_user` used to be accepted by the eight gates that had no TTY check, and
+  now exits 1 naming `--yes`. Data-destroying work should not proceed on a `y`
+  that arrived from a pipe rather than a person, and `--yes` has always been
+  the documented scriptable path — but any script relying on the old behaviour
+  fails loudly and needs `--yes` instead. Sites that already refused a piped
+  answer (`dp db query`'s write guard, the AWS secrets writes) are unchanged.
+
+- `line-length = 88` is enforced. It was configured but unchecked, because
+  pycodestyle was not among the selected rule sets, and the tree had drifted
+  to 164 violations across 71 unformatted files. The repo is now
+  `ruff format`-clean and CI keeps it that way.
+
+## 0.1.0
+
+Initial public release.
+
+- Optional per-area dependency extras (`dataplat[db,ingest,bi,cloud,all]`)
+  with auto-detection: `dp config sync` installs what your enabled areas
+  need, and hitting a stubbed area offers to install its extra and re-run
+  your command.
+
+- `dp` command with areas per component type: `db`, `ingest` (Airbyte),
+  `bi` (Superset), `cloud` (AWS), `ci` (GitHub runners), plus `status`,
+  `open`, and `config`.
+- Fully config-driven database targets via `DP_TARGETS` and per-target
+  `<NAME>_*` environment variables (Postgres and Redshift engines).
+- `.envrc` loading with a global config link (`dp config init`).

{dataplat-0.1.0 → dataplat-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataplat
-Version: 0.1.0
+Version: 0.2.0
 Summary: One command to manage any shape of data platform: databases, ingestion, BI, cloud, and CI.
 Project-URL: Homepage, https://github.com/hanslemm/dataplat
 Project-URL: Repository, https://github.com/hanslemm/dataplat
@@ -15,11 +15,12 @@ Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
 Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Database
 Classifier: Topic :: System :: Systems Administration
 Classifier: Topic :: Utilities
-Requires-Python: >=3.13
+Requires-Python: >=3.12
 Requires-Dist: rich>=14.2.0
 Requires-Dist: typer>=0.21.1
 Provides-Extra: all
@@ -93,6 +94,8 @@ dp
 
 ## Installation
 
+Requires Python 3.12 or newer.
+
 ```bash
 uv tool install "dataplat[all]"     # recommended: everything
 # or
@@ -133,6 +136,11 @@ and re-runs your original command. Non-interactive sessions never install
 silently — they print the command and exit instead. `dp config doctor`
 also reports per-area dependency status.
 
+Either path only ever *adds* to your install: the command is pinned to the
+`dataplat` version you already run, so installing an extra never upgrades the
+tool underneath you, and it carries your existing extras along, so adding
+`db` cannot drop an `ingest` you already had.
+
 ## Quick start
 
 1. Declare your database targets — any names you like:
@@ -180,11 +188,21 @@ already set in your shell. Lookup order:
 1. `DP_ENVRC_PATH`
 2. `~/.config/dataplat/.envrc` (the global link — set it with `dp config init`)
 3. `.envrc` in the current directory
+4. the `.envrc` beside a development checkout of dataplat itself
+
+Candidate 3 makes every command sensitive to where you run it: standing in a
+cloned repo points `dp` at whatever host and credentials that repo's `.envrc`
+exports. `dp config show` and `dp config doctor` always name the active file
+*and* which candidate produced it, and warn when it came from the current
+directory. Set `DP_ENVRC_ALLOW_CWD=0` to drop that candidate entirely and
+rely only on the global link you chose.
 
 ## Configuration reference
 
 | Variable | Purpose |
 | --- | --- |
+| `DP_ENVRC_PATH` | Explicit `.envrc` to load, ahead of every other candidate. |
+| `DP_ENVRC_ALLOW_CWD` | Set to `0` to stop picking up `.envrc` from the current directory. |
 | `DP_TARGETS` | Comma-separated DB target names (e.g. `warehouse,lake`). |
 | `DP_DEFAULT_TARGET` | Target used when `--target` is omitted (default: first of `DP_TARGETS`). |
 | `<NAME>_ENGINE` | `postgresql` (default) or `redshift`, per target. |
@@ -312,13 +330,19 @@ cd dataplat
 uv sync --group dev --all-extras
 uv run pytest
 uv run ruff check .
+uv run ruff format --check .
 uv run mypy dataplat
 ```
 
+CI runs those four across Python 3.12 and 3.13 — the floor the wheel
+advertises as well as the pinned dev version.
+
 ## Releasing
 
-Tag `X.Y.Z` (bare semver, no `v` prefix) on `main`; GitHub Actions builds
-and publishes to PyPI via Trusted Publishing. Commits follow
+Bump `[project].version`, then tag `X.Y.Z` (bare semver, no `v` prefix) on
+`main`. The release workflow refuses to publish unless the tag matches that
+version, and runs the full check matrix first; only then does GitHub Actions
+build and publish to PyPI via Trusted Publishing. Commits follow
 [Conventional Commits](https://www.conventionalcommits.org/).
 
 ## License

{dataplat-0.1.0 → dataplat-0.2.0}/README.md

@@ -49,6 +49,8 @@ dp
 
 ## Installation
 
+Requires Python 3.12 or newer.
+
 ```bash
 uv tool install "dataplat[all]"     # recommended: everything
 # or
@@ -89,6 +91,11 @@ and re-runs your original command. Non-interactive sessions never install
 silently — they print the command and exit instead. `dp config doctor`
 also reports per-area dependency status.
 
+Either path only ever *adds* to your install: the command is pinned to the
+`dataplat` version you already run, so installing an extra never upgrades the
+tool underneath you, and it carries your existing extras along, so adding
+`db` cannot drop an `ingest` you already had.
+
 ## Quick start
 
 1. Declare your database targets — any names you like:
@@ -136,11 +143,21 @@ already set in your shell. Lookup order:
 1. `DP_ENVRC_PATH`
 2. `~/.config/dataplat/.envrc` (the global link — set it with `dp config init`)
 3. `.envrc` in the current directory
+4. the `.envrc` beside a development checkout of dataplat itself
+
+Candidate 3 makes every command sensitive to where you run it: standing in a
+cloned repo points `dp` at whatever host and credentials that repo's `.envrc`
+exports. `dp config show` and `dp config doctor` always name the active file
+*and* which candidate produced it, and warn when it came from the current
+directory. Set `DP_ENVRC_ALLOW_CWD=0` to drop that candidate entirely and
+rely only on the global link you chose.
 
 ## Configuration reference
 
 | Variable | Purpose |
 | --- | --- |
+| `DP_ENVRC_PATH` | Explicit `.envrc` to load, ahead of every other candidate. |
+| `DP_ENVRC_ALLOW_CWD` | Set to `0` to stop picking up `.envrc` from the current directory. |
 | `DP_TARGETS` | Comma-separated DB target names (e.g. `warehouse,lake`). |
 | `DP_DEFAULT_TARGET` | Target used when `--target` is omitted (default: first of `DP_TARGETS`). |
 | `<NAME>_ENGINE` | `postgresql` (default) or `redshift`, per target. |
@@ -268,13 +285,19 @@ cd dataplat
 uv sync --group dev --all-extras
 uv run pytest
 uv run ruff check .
+uv run ruff format --check .
 uv run mypy dataplat
 ```
 
+CI runs those four across Python 3.12 and 3.13 — the floor the wheel
+advertises as well as the pinned dev version.
+
 ## Releasing
 
-Tag `X.Y.Z` (bare semver, no `v` prefix) on `main`; GitHub Actions builds
-and publishes to PyPI via Trusted Publishing. Commits follow
+Bump `[project].version`, then tag `X.Y.Z` (bare semver, no `v` prefix) on
+`main`. The release workflow refuses to publish unless the tag matches that
+version, and runs the full check matrix first; only then does GitHub Actions
+build and publish to PyPI via Trusted Publishing. Commits follow
 [Conventional Commits](https://www.conventionalcommits.org/).
 
 ## License

dataplat-0.2.0/dataplat/__init__.py

@@ -0,0 +1,27 @@
+"""dataplat — one command to manage any shape of data platform."""
+
+from __future__ import annotations
+
+from typing import Any
+
+__all__ = ["__version__"]
+
+
+def __getattr__(name: str) -> Any:
+    """Serve ``__version__`` from the installed distribution, on demand.
+
+    A literal here would be a second place to edit at release time, and the
+    kind that goes stale silently: nothing in the package reads it, so a
+    mismatch with ``[project].version`` would never surface. Reading the
+    metadata keeps pyproject the only source of truth, and doing it in
+    ``__getattr__`` (PEP 562) keeps ``importlib.metadata`` off the import path
+    of every ``dp`` invocation that never asks.
+    """
+    if name == "__version__":
+        from importlib.metadata import PackageNotFoundError, version
+
+        try:
+            return version("dataplat")
+        except PackageNotFoundError:  # running from a bare source tree
+            return "unknown"
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

dataplat-0.2.0/dataplat/cli/_lazy.py

@@ -0,0 +1,108 @@
+"""Mount areas by name; import one only when it is actually run.
+
+``main`` used to call :func:`~dataplat.core.registry.load_app` for every area
+at import time, so ``dp --version`` paid for psycopg, httpx and croniter
+(~250 ms of a ~280 ms run) just to print one line. Here an area is mounted as
+an empty placeholder group carrying nothing but the name and help text the
+registry already knows, and :class:`LazyRootGroup` swaps in the real Typer app
+— or the missing-deps stub — when click resolves that area, and only then.
+
+Placeholders are mounted through ``add_typer`` rather than injected into the
+click group, so command order, ``--help``, "did you mean" suggestions and
+top-level completion keep coming out of Typer's own machinery instead of a
+reimplementation of it, and nothing else in the CLI has to know that areas are
+lazy. Those paths are all import-free (verified: ``dp --help`` and ``dp <TAB>``
+leave psycopg unimported). Completing *inside* an area — ``dp db <TAB>`` — does
+import it, because the subcommand list it has to offer is the area's own; that
+is the one non-dispatch path that pays, and there is no way around it.
+
+No direct click import here either — typer vendors click, so only the
+``TyperGroup`` surface is safe to rely on (see :mod:`dataplat.cli._missing`).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import typer
+from typer.core import TyperGroup
+from typer.main import get_group
+
+from dataplat.core.deps import ready
+from dataplat.core.registry import AreaMount, area_by_name, load_app, mount_help
+
+__all__ = [
+    "AreaPlaceholderGroup",
+    "LazyRootGroup",
+    "area_command",
+    "area_placeholder",
+]
+
+
+class AreaPlaceholderGroup(TyperGroup):
+    """Stands in for an area nobody has imported yet.
+
+    Deliberately empty: it is only ever asked to describe itself (name, help
+    text, hidden), and :meth:`LazyRootGroup.resolve_command` replaces it before
+    anything can be invoked through it.
+    """
+
+
+def area_placeholder(mount: AreaMount) -> typer.Typer:
+    """A command-less Typer app holding just ``mount``'s name and help line."""
+    return typer.Typer(
+        name=mount.name,
+        help=mount_help(mount),
+        cls=AreaPlaceholderGroup,
+    )
+
+
+def area_command(mount: AreaMount) -> typer.Typer:
+    """``mount``'s real Typer app, or the stub that offers to install its extra.
+
+    Readiness is decided here rather than at mount time, so an area whose extra
+    is missing costs nothing until someone runs it — and it is decided from
+    ``mount.deps``, so an area that is not one of the built-ins still works.
+    """
+    if mount.deps is None or ready(mount.deps):
+        return load_app(mount)
+    # Imported on this branch only: the stub pulls in subprocess, which costs
+    # more to import (~5 ms) than every readiness check in the CLI together.
+    from dataplat.cli._missing import build_missing_deps_app
+
+    return build_missing_deps_app(mount)
+
+
+class LazyRootGroup(TyperGroup):
+    """Root group that resolves an area's real app the first time it is used."""
+
+    def resolve_command(self, ctx: Any, args: Any) -> Any:
+        """Import the area behind a placeholder, now that it is needed.
+
+        Resolution is what "the user is reaching into this area" looks like to
+        click. ``--help``, typo suggestions and top-level completion never come
+        through here — they read the placeholder and stay import-free. Shell
+        completion *descending* into an area does come through here, with
+        ``ctx.resilient_parsing`` set, and must still import: the completions it
+        owes the shell are the area's own subcommands.
+        """
+        name, cmd, rest = super().resolve_command(ctx, args)
+        if isinstance(cmd, AreaPlaceholderGroup) and name is not None:
+            cmd = self._load_area(name)
+        return name, cmd, rest
+
+    def _load_area(self, name: str) -> Any:
+        mount = area_by_name(name)
+        # The placeholder exists because a mount was registered under this name.
+        assert mount is not None, f"no registered area named {name!r}"
+        # get_group, not get_command: the latter bolts the completion options
+        # onto whatever it builds, which add_typer never did for an area.
+        group = get_group(area_command(mount))
+        # add_typer(name=...) used to name the mounted group; get_group takes
+        # the name off the area's own Typer, so keep the mount authoritative —
+        # a third-party area may well name its app something else.
+        group.name = name
+        # Cached in place: replacing the placeholder keeps command order intact
+        # and means one process imports an area at most once.
+        self.commands[name] = group
+        return group