pgiscluster 0.2.0__py3-none-any.whl

pgiscluster/__init__.py

@@ -0,0 +1,7 @@
+"""Module for pgiscluster."""
+
+from pgiscluster.connector import HostAdminDBConnector, HostDBConnector
+
+__version__ = "0.2.0"
+
+__all__ = ["HostAdminDBConnector", "HostDBConnector"]

pgiscluster/connector.py

@@ -0,0 +1,160 @@
+import base64
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+import geopandas as gpd
+import kubernetes as k8s
+import sqlalchemy as sql
+from sqlalchemy import event
+
+
+class DBConnector(ABC):
+    """Assembles a PostgreSQL connection address, then queries the database"""
+
+    def __init__(
+        self,
+        database: str = "data_science",
+        target_resolver: Callable[[], tuple[str, int]] | None = None,
+    ) -> None:
+        self.drivername = "postgresql+psycopg"
+        self.database = database
+        self.host, self.port = (target_resolver or self._get_target)()
+        self.engine = self._create_engine()
+
+    def _get_target(self) -> tuple[str, int]:
+        """Returns the (host, port) to connect to, resolved live from the cluster's CiliumLocalRedirectPolicy."""
+        k8s.config.load_kube_config()
+        crd_list = k8s.client.ApiextensionsV1Api().list_custom_resource_definition()
+        assert isinstance(crd_list, k8s.client.V1CustomResourceDefinitionList)
+        assert crd_list.items is not None
+
+        local_policy = next(
+            crd
+            for crd in crd_list.items
+            if crd.spec.names.kind == "CiliumLocalRedirectPolicy"
+        )
+        group = local_policy.spec.group
+        plural = local_policy.spec.names.plural
+        served_versions = [
+            version for version in local_policy.spec.versions if version.served
+        ]
+        if not served_versions:
+            raise RuntimeError(
+                f"{local_policy.metadata.name} CRD has no served version"
+            )
+        storage_version_options = (
+            version for version in served_versions if version.storage
+        )
+        storage_version = next(storage_version_options, None)
+        if storage_version is None:
+            raise RuntimeError(
+                f"{local_policy.metadata.name} CRD has no storage version among its served versions"
+            )
+        if next(storage_version_options, None) is not None:
+            raise RuntimeError(
+                f"{local_policy.metadata.name} CRD has multiple storage versions"
+            )
+        raw_result = k8s.client.CustomObjectsApi().list_cluster_custom_object(
+            group=group,
+            plural=plural,
+            version=storage_version.name,
+            label_selector="app.kubernetes.io/name=postgis-cluster",
+        )
+        # kubernetes client lacks type hints until v37; drop this on its release
+        assert isinstance(raw_result, dict)
+        result: dict[str, Any] = raw_result
+        if len(result["items"]) != 1:
+            raise RuntimeError(
+                f"expected exactly one CiliumLocalRedirectPolicy matching label "
+                f"app.kubernetes.io/name=postgis-cluster, found {len(result['items'])}"
+            )
+        address_matcher = result["items"][0]["spec"]["redirectFrontend"][
+            "addressMatcher"
+        ]
+        host = address_matcher["ip"]
+        port = int(address_matcher["toPorts"][0]["port"])
+        return (host, port)
+
+    @abstractmethod
+    def _get_credentials(self) -> tuple[str, str]:
+        """Returns a fresh (username, password) pair."""
+        raise NotImplementedError
+
+    def _create_engine(self) -> sql.Engine:
+        """Uses DBConnector attributes to return database address."""
+        engine = sql.create_engine(f"{self.drivername}://")
+
+        @event.listens_for(engine, "do_connect")
+        def _inject_credentials(*args):
+            *_, cparams = args
+            username, password = self._get_credentials()
+            cparams["user"] = username
+            cparams["password"] = password
+            cparams["host"] = self.host
+            cparams["port"] = self.port
+            cparams["dbname"] = self.database
+
+        return engine
+
+    def db_query(
+        self,
+        query_file: str | Path | None = None,
+        *,
+        query: str | None = None,
+        geom_col: str,
+    ) -> gpd.GeoDataFrame:
+        """Runs a SQL query against the PostGIS database, returning a GeoDataFrame.
+
+        Provide query_file (a path to a .sql file) or query (a literal SQL string).
+        """
+        if query is None:
+            if query_file is None:
+                raise ValueError("either query_file or query must be given")
+            query = Path(query_file).read_text()
+        geodataframe = gpd.read_postgis(
+            sql=query,
+            con=self.engine,
+            # NOTE: GeoDataFrame can only analyze one geometry column at a time
+            geom_col=geom_col,
+        )
+        return geodataframe
+
+
+class HostDBConnector(DBConnector):
+    """Connects to the database from outside the cluster, using
+    the VSO-synced dynamic-credentials Secret."""
+
+    SECRET_NAME = "postgis-app-dynamic-credentials"
+    SECRET_NAMESPACE = "databases"
+
+    def _get_credentials(self) -> tuple[str, str]:
+        """Returns a fresh (username, password) pair decoded from the VSO-synced Secret."""
+        secret = k8s.client.CoreV1Api().read_namespaced_secret(
+            self.SECRET_NAME, self.SECRET_NAMESPACE
+        )
+        assert isinstance(secret, k8s.client.V1Secret)
+        assert secret.data is not None
+        missing_keys = [
+            key for key in ("username", "password") if key not in secret.data
+        ]
+        if missing_keys:
+            raise RuntimeError(
+                f"{self.SECRET_NAME} Secret missing key(s): {', '.join(missing_keys)}"
+            )
+        username = base64.b64decode(secret.data["username"]).decode()
+        password = base64.b64decode(secret.data["password"]).decode()
+        return (username, password)
+
+
+class HostAdminDBConnector(HostDBConnector):
+    """Connects to the database from outside the cluster using the Postgres
+    superuser credential, for schema/DDL changes rather than routine queries.
+
+    Unlike HostDBConnector's dynamic lease, this credential is static
+    (doesn't expire or rotate) and unscoped to a single database — it's the
+    same credential CloudNativePG itself uses for cluster bootstrap and
+    reconciliation. Use deliberately, not as a default."""
+
+    SECRET_NAME = "postgis-app-credentials"

pgiscluster/features/__init__.py

@@ -0,0 +1 @@
+"""Scripts to turn raw data into features for modeling."""

pgiscluster-0.2.0.dist-info/METADATA

@@ -0,0 +1,119 @@
+Metadata-Version: 2.5
+Name: pgiscluster
+Version: 0.2.0
+Summary: Python classes intended to orchestrate Extract, Transform, & Load (ETL) Data Pipelines and Machine Learning (ML) tasks.
+Project-URL: Homepage, https://github.com/DragonBishop/pgiscluster
+Project-URL: Repository, https://github.com/DragonBishop/pgiscluster
+Project-URL: Issues, https://github.com/DragonBishop/pgiscluster/issues
+Author-email: DragonBishop <276789280+DragonBishop@users.noreply.github.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.14.7
+Requires-Dist: geopandas>=1.1.4
+Requires-Dist: hvac>=2.4.0
+Requires-Dist: kubernetes>=36.0.3
+Requires-Dist: pandas>=3.0.5
+Requires-Dist: psycopg[binary]>=3.3.4
+Requires-Dist: sqlalchemy>=2.0.52
+Provides-Extra: common
+Requires-Dist: ipykernel>=7.3.0; extra == 'common'
+Requires-Dist: prefect>=3.8.5; extra == 'common'
+Provides-Extra: etl
+Requires-Dist: dbt-core>=1.12.3; extra == 'etl'
+Requires-Dist: dbt-postgres>=1.11.0; extra == 'etl'
+Requires-Dist: dlt>=1.30.0; extra == 'etl'
+Requires-Dist: ipykernel>=7.3.0; extra == 'etl'
+Requires-Dist: prefect>=3.8.5; extra == 'etl'
+Provides-Extra: ml
+Requires-Dist: bentoml>=1.2.0; extra == 'ml'
+Requires-Dist: ipykernel>=7.3.0; extra == 'ml'
+Requires-Dist: mlflow>=3.16.0; extra == 'ml'
+Requires-Dist: prefect>=3.8.5; extra == 'ml'
+Requires-Dist: scikit-learn>=1.3.2; extra == 'ml'
+Description-Content-Type: text/markdown
+
+# pgiscluster
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/DragonBishop/pgiscluster/blob/main/LICENSE)
+[![Python 3.14+](https://img.shields.io/badge/python-3.14%2B-blue.svg)](https://www.python.org/downloads/)
+
+Python classes for connecting to and querying a PostGIS database, with credentials resolved dynamically at runtime.
+
+**Status: Alpha.** Only `HostDBConnector` is implemented. `ClusterDBConnector` is planned but not built. Its default address and credential discovery assume specific cluster infrastructure, but both are plain, overridable methods.
+
+## Key features
+
+- Fetches credentials from a Kubernetes Secret on every connection, so rotation is picked up automatically.
+- Resolves the database address live from the cluster.
+- Query results returned as GeoDataFrames for downstream GIS/analytics work.
+
+## Install
+
+```bash
+pip install pgiscluster
+```
+
+## Requirements
+
+- Python >= 3.14.7
+- A `kubectl` context with:
+  - Cluster-scoped read access to CustomResourceDefinitions and their custom objects, for address discovery.
+  - Read access to Secrets in the namespace holding the database credentials.
+- A reachable host:port for the database. By default, resolved via a Cilium `CiliumLocalRedirectPolicy` labeled `app.kubernetes.io/name=postgis-cluster`.
+- A Kubernetes Secret with base64-encoded `username`/`password` keys for the database login.
+- See [Configuration](#configuration) for alternatives to defaults.
+
+## Usage
+
+```python
+from pgiscluster import HostDBConnector
+
+connector = HostDBConnector()
+df = connector.db_query(
+    "queries/nearby.sql", geom_col="geom"
+)  # from a .sql file (default)
+df = connector.db_query(query="SELECT * FROM my_table", geom_col="geom")  # literal SQL
+```
+
+`HostDBConnector` resolves the database host/port at construction, then fetches a fresh username/password pair on every connection attempt.
+
+### Configuration
+
+#### `HostDBConnector`
+
+`HostDBConnector(database="data_science", target_resolver=None)`: `database` is the target database name, defaulting to `"data_science"`.
+
+By default, `HostDBConnector` resolves host/port from the cluster's `CiliumLocalRedirectPolicy`, reading the address from its `spec.redirectFrontend.addressMatcher`. For a different mechanism (a NodePort/LoadBalancer Service, a port-forward, a socat proxy), override `_get_target` in a subclass to reuse the same logic across many instances, or pass `target_resolver`, a zero-argument callable returning `(host, port)`, for a one-off:
+
+```python
+connector = HostDBConnector(target_resolver=lambda: ("localhost", 5432))
+```
+
+Credentials come from a Kubernetes Secret. Its name and namespace are set by the class attributes `HostDBConnector.SECRET_NAME` and `HostDBConnector.SECRET_NAMESPACE`. You can override them to point elsewhere. The reference cluster populates that Secret via the Vault Secrets Operator, syncing a dynamically-generated HashiCorp Vault credential. However, any mechanism works as long as the Secret has `username`/`password` keys.
+
+`db_query(query_file=None, *, query=None, geom_col)` runs a SQL query and returns a [GeoDataFrame](https://geopandas.org/), decoding one geometry column per call. One of either `query_file` (a path to a `.sql` file, the default), or `query`, a literal SQL string.
+
+#### `HostAdminDBConnector`
+
+A `HostDBConnector` subclass that reads the cluster's Postgres **superuser** credential from a static secret, instead of the scoped `app_readwrite` dynamic lease, for schema/DDL changes:
+
+```python
+from pgiscluster import HostAdminDBConnector
+
+connector = HostAdminDBConnector()
+```
+
+## License
+
+See [MIT LICENSE](https://github.com/DragonBishop/pgiscluster/blob/main/LICENSE).
+
+## Links
+
+- [Source](https://github.com/DragonBishop/pgiscluster)
+- [Issues](https://github.com/DragonBishop/pgiscluster/issues)
+- [Changelog](https://github.com/DragonBishop/pgiscluster/blob/main/CHANGELOG.md)
+
+`pgiscluster` was originally developed inside the [Data Science Cluster](https://github.com/DragonBishop/data_science_cluster) repo, the reference cluster its defaults are built against. If you want to contribute, open a development branch and then submit a pull request.

pgiscluster-0.2.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+pgiscluster/__init__.py,sha256=A5G9CiOBIzqhdZL5jS3GZSFxgywYqt02u8I1eGRXcpM,181
+pgiscluster/connector.py,sha256=PUOvVhNWATjUtXRnAImVIu2CSerqo1em5QLKf7Y6hJ8,6188
+pgiscluster/features/__init__.py,sha256=8-i_XeuERLebwhYIRb7ZxZRmB_k2LvZf21vZd4IrR40,59
+pgiscluster-0.2.0.dist-info/METADATA,sha256=6kqtV9cl7AAv7JD6naPHNwvHDs36Xp-15FxAPJra8b4,5754
+pgiscluster-0.2.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+pgiscluster-0.2.0.dist-info/licenses/LICENSE,sha256=BXfwJsVlvEUEUc4kokFcApRorrBWqnukpLLtinVJuS0,1069
+pgiscluster-0.2.0.dist-info/RECORD,,

pgiscluster-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pgiscluster-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 DragonBishop
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.