nc-db 0.1.3__py3-none-any.whl

nc_db/README.md

@@ -0,0 +1,156 @@
+# nc_db
+PostgreSQL schema, lightweight Python DB client, tests, and Kubernetes manifests for the Neural Context project.
+
+This repository provides:
+- A thin, typed Python client for JSONB-first tables (insert/update/select/delete) under a configurable schema.
+- Test suite that spins up isolated schemas to validate behavior and edge cases.
+- Kubernetes manifests (Kustomize) to run Postgres locally or in-cluster, plus a safe reset script.
+
+## Contents
+
+```
+src/db/           # Python client: Db, DbConfig, SQL helpers
+src/tests/        # Pytest suite and fixtures (isolated schemas)
+k8s/              # K8s StatefulSet, init SQL, and ops scripts
+pyproject.toml    # Poetry + pytest configuration
+.github/          # Issue and PR templates
+README.md         # This file (schema diagram below)
+```
+
+## Prerequisites
+
+- Python 3.12+
+- PostgreSQL 15+ available locally, or run via the provided K8s manifests
+
+Default development credentials used in examples/tests:
+`host=localhost, port=5432, db=nc-data, user=nc_admin, password=bob`.
+
+## Quick Start
+
+1) Install dependencies with Poetry:
+
+```bash
+poetry install
+```
+
+2) Ensure Postgres is reachable using the defaults above. You can run it via Kubernetes (see below) or your local Postgres.
+
+3) Smoke test the Python client:
+
+```bash
+poetry run python -c "from src.db.db import Db; print(Db().get('scg', ids=[]))"
+```
+
+## Running Tests
+
+```bash
+kubectl apply -k k8s/postgres/overlays/ephemeral
+kubectl -n nc_db wait --for=condition=ready pod -l app=nc-data --timeout=90s
+kubectl -n nc_db port-forward svc/postgres 5432:5432
+```
+
+```bash
+poetry run pytest -q
+```
+
+- To run a single test: `poetry run pytest -k test_update_merge -q`
+- Tests create per-test schemas and tables; they do not require pre-existing app tables.
+
+## Python Client Overview
+
+The client favors tables shaped as `(id BIGSERIAL PRIMARY KEY, body JSONB NOT NULL)` and enforces `body.id == id` on writes/reads.
+
+Basic usage:
+
+```python
+from nc_db.db.db import Db
+from nc_db.db.DbConfig import DbConfig
+
+db = Db(schema="nc", config=DbConfig(
+    user="my_user", password="my_password", host="localhost", port=5432, dbname="my_database"
+))
+
+# Insert with or without explicit id
+id1 = db.insert("scg", {"body": {"name": "alpha"}})
+id2 = db.insert("scg", {"body": {"id": 42, "name": "explicit"}})
+
+# Update (merge JSON into existing body or replace entirely)
+db.update("scg", ids=[id1], items=[{"body": {"name": "alpha-2"}}], mode="merge")
+
+# Read (optionally map rows to models)
+rows = db.get("scg")  # [{"id": 1, "body": {...}}, ...]
+
+# Delete
+db.delete("scg", [id2])
+```
+
+Key features:
+- Insert trusts a provided id if present; otherwise uses the table sequence and normalizes `body.id`.
+- Update supports `mode="merge"` (default) and `mode="replace"`, always ensuring `body.id` matches the row id.
+- Get returns rows as dictionaries or via a mapper function to your own model types.
+- Delete single or all rows, plus a helper to `TRUNCATE ... RESTART IDENTITY`.
+
+## Configuration
+
+Use `DbConfig` or environment variables to control connection pooling and DSN; defaults are suitable for local development.
+
+```python
+from nc_db.db.DbConfig import DbConfig
+
+config = DbConfig(
+    user="your_username",
+    password="your_password",
+    host="localhost",
+    port=5432,
+    dbname="your_database",
+)
+```
+OR
+
+```bash
+export PGUSER="your_username"
+export PGPASSWORD="your_password"
+export PGHOST=localhost
+export PGPORT=5432
+export PGDATABASE="youd_database"
+
+```
+
+## Kubernetes (Optional)
+
+Run a local Postgres via Kustomize overlays.
+
+Ephemeral (emptyDir) instance:
+
+```bash
+kubectl apply -k k8s/postgres/overlays/ephemeral
+kubectl -n nc_db wait --for=condition=ready pod -l app=nc-data --timeout=90s
+kubectl -n nc_db port-forward svc/postgres 5432:5432
+```
+
+Persistent storage overlay is also available under `k8s/postgres/overlays/persistent`.
+
+To safely reset a StatefulSet (DANGEROUS — wipes data), use:
+
+```bash
+python k8s/scripts/full_pod_reset.py --yes
+```
+
+Review the script before use; it scales down, deletes the PVC (and optionally PV), and scales back up.
+
+## Development Conventions
+
+- Python 3.12, type hints, PEP 8; keep imports sorted and code minimal.
+- Modules: snake_case; Classes: CamelCase; variables/functions: snake_case.
+- Tests live in `src/tests`; name files/functions `test_*` and focus on behavior.
+- See `.github/PULL_REQUEST_TEMPLATE.md` for PR expectations.
+
+## Schema Diagram
+
+Note: the init SQL currently provisions `scg` and `healthcheck`
+
+## Security & License
+
+- Do not commit real secrets. The defaults in code and K8s manifests are for local development only.
+- Use `DbConfig` or environment variables to specify connection details.
+- License: Proprietary (see `pyproject.toml`).

nc_db/k8s/postgres/base/init/01-create-schema.sql

@@ -0,0 +1,8 @@
+CREATE SCHEMA IF NOT EXISTS nc;
+
+CREATE TABLE IF NOT EXISTS nc.scg (
+	id BIGSERIAL PRIMARY KEY,
+	body JSONB NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_data_body_gin on nc.scg USING GIN (body);

nc_db/k8s/postgres/base/init/02-grants.sql

@@ -0,0 +1,16 @@
+-- Revoke overly broad PUBLIC defaults and grant to the application role
+
+-- Revoke defaults for future objects
+ALTER DEFAULT PRIVILEGES IN SCHEMA "nc" REVOKE ALL ON TABLES FROM PUBLIC;
+ALTER DEFAULT PRIVILEGES IN SCHEMA "nc" REVOKE ALL ON SEQUENCES FROM PUBLIC;
+
+-- Revoke on existing objects (no-op if none)
+REVOKE ALL ON ALL TABLES IN SCHEMA nc FROM PUBLIC;
+REVOKE ALL ON ALL SEQUENCES IN SCHEMA nc FROM PUBLIC;
+
+-- Grant defaults to nc_app for future objects (owner-dependent)
+ALTER DEFAULT PRIVILEGES IN SCHEMA "nc"
+GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO nc_app;
+
+ALTER DEFAULT PRIVILEGES IN SCHEMA "nc"
+GRANT USAGE, SELECT ON SEQUENCES TO nc_app;

nc_db/k8s/postgres/base/init/10-healthcheck.sql

@@ -0,0 +1,10 @@
+CREATE SCHEMA IF NOT EXISTS nc;
+
+CREATE TABLE IF NOT EXISTS nc.healthcheck (
+  id int PRIMARY KEY,
+  ok boolean NOT NULL
+);
+
+INSERT INTO nc.healthcheck (id, ok)
+VALUES (1, true)
+ON CONFLICT (id) DO NOTHING;

nc_db/k8s/postgres/base/kustomization.yaml

@@ -0,0 +1,31 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: nc_db
+
+secretGenerator:
+  # Cluster/postgres superuser creds
+  - name: postgres-credentials-root
+    envs:
+      - secrets/postgres.root.env
+
+  # App/user creds (what your app uses to connect)
+  - name: postgres-credentials-app
+    envs:
+      - secrets/postgres.app.env
+
+configMapGenerator:
+  - name: pg-init-sql
+    files:
+      - init/00-create-app-role.sql
+      - init/01-create-schema.sql
+      - init/02-grants.sql
+      - init/10-healthcheck.sql
+
+# Keep the suffix hash enabled so rolling updates occur when content changes
+generatorOptions:
+  disableNameSuffixHash: false
+
+resources:
+  - namespace.yaml
+  - service.yaml
+  - statefulset.yaml

nc_db/k8s/postgres/base/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nc_db

nc_db/k8s/postgres/base/pg-app-bootstrap.yaml

@@ -0,0 +1,50 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: pg-app-bootstrap
+  namespace: nc_db
+type: Opaque
+stringData:
+  20-create-app-user.sh: |
+    #!/bin/bash
+    set -euo pipefail
+
+    echo "[initdb] creating / ensuring application role ${APP_DB_USER}"
+
+    psql -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}" --dbname "${POSTGRES_DB}" <<-EOSQL
+      DO \$\$
+      BEGIN
+          IF NOT EXISTS (
+              SELECT 1 FROM pg_roles WHERE rolname = '${APP_DB_USER}'
+          ) THEN
+              EXECUTE format(
+                  'CREATE ROLE %I LOGIN PASSWORD %L',
+                  '${APP_DB_USER}',
+                  '${APP_DB_PASSWORD}'
+              );
+          END IF;
+      END
+      \$\$;
+
+      -- ensure schema exists
+      CREATE SCHEMA IF NOT EXISTS ${APP_DB_SCHEMA};
+
+      -- make sure app user can actually use the DB
+      GRANT CONNECT ON DATABASE "${APP_DB_NAME}" TO ${APP_DB_USER};
+
+      GRANT USAGE ON SCHEMA ${APP_DB_SCHEMA} TO ${APP_DB_USER};
+
+      GRANT SELECT, INSERT, UPDATE, DELETE
+        ON ALL TABLES IN SCHEMA ${APP_DB_SCHEMA}
+        TO ${APP_DB_USER};
+
+      GRANT USAGE, SELECT
+        ON ALL SEQUENCES IN SCHEMA ${APP_DB_SCHEMA}
+        TO ${APP_DB_USER};
+
+      ALTER DEFAULT PRIVILEGES IN SCHEMA ${APP_DB_SCHEMA}
+        GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ${APP_DB_USER};
+
+      ALTER DEFAULT PRIVILEGES IN SCHEMA ${APP_DB_SCHEMA}
+        GRANT USAGE, SELECT ON SEQUENCES TO ${APP_DB_USER};
+    EOSQL

nc_db/k8s/postgres/base/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+spec:
+  type: ClusterIP
+  selector:
+    app: nc-data
+  ports:
+    - name: pg
+      port: 5432
+      targetPort: 5432

nc_db/k8s/postgres/base/statefulset.yaml

@@ -0,0 +1,88 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: postgres
+  namespace: nc_db
+spec:
+  serviceName: postgres
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nc-data
+  template:
+    metadata:
+      labels:
+        app: nc-data
+    spec:
+      containers:
+        - name: postgres
+          image: postgres:15
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 5432
+              name: pg
+
+          # Load secrets into env so init script can read them
+          envFrom:
+            - secretRef:
+                name: postgres-credentials-root        # POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB
+            - secretRef:
+                name: postgres-credentials-app         # APP_DB_USER, APP_DB_PASSWORD, APP_DB_NAME, APP_DB_SCHEMA
+
+          # Non-secret config
+          env:
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
+
+          # NOTE: lifecycle: postStart was intentionally removed
+
+          readinessProbe:
+            tcpSocket:
+              port: 5432
+            initialDelaySeconds: 5
+            periodSeconds: 5
+
+          livenessProbe:
+            tcpSocket:
+              port: 5432
+            initialDelaySeconds: 10
+            periodSeconds: 10
+
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/postgresql/data
+
+            # merged init dir: configmap SQL + secret script
+            - name: init-bootstrap
+              mountPath: /docker-entrypoint-initdb.d
+              readOnly: true
+
+      volumes:
+        - name: init-bootstrap
+          projected:
+            sources:
+              - configMap:
+                  name: pg-init-sql
+                  items:
+                    - key: 01-create-schema.sql
+                      path: 01-create-schema.sql
+                    - key: 02-grants.sql
+                      path: 02-grants.sql
+                    - key: 10-healthcheck.sql
+                      path: 10-healthcheck.sql
+              - secret:
+                  name: pg-app-bootstrap
+                  items:
+                    - key: 20-create-app-user.sh
+                      path: 20-create-app-user.sh
+
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        storageClassName: microk8s-hostpath
+        resources:
+          requests:
+            storage: 5Gi

nc_db/k8s/postgres/overlays/ephemeral/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: nc_db
+resources:
+  - ../../base
+patches:
+  - path: patch-ephemeral-emptydir.yaml

nc_db/k8s/postgres/overlays/ephemeral/patch-ephemeral-emptydir.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: postgres
+spec:
+  template:
+    spec:
+      volumes:
+        - name: init-sql
+          configMap:
+            name: pg-init-sql
+            defaultMode: 0444
+        - name: data
+          emptyDir: {}
+  volumeClaimTemplates: []  # remove PVCs for ephemeral runs

nc_db/k8s/postgres/overlays/persistent/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: nc_db
+resources:
+  - ../../base
+patches:
+  - path: patch-persistent-storageclass.yaml

nc_db/k8s/postgres/overlays/persistent/patch-persistent-storageclass.yaml

@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: postgres
+spec:
+  volumeClaimTemplates: []   # disable PVC template from base
+  template:
+    spec:
+      volumes:
+        - name: data
+          hostPath:
+            path: /home/chale/nc_db/postgress-store/
+            type: DirectoryOrCreate
+        - name: init-sql
+          configMap:
+            name: pg-init-sql
+            defaultMode: 0444

nc_db/k8s/scripts/full_pod_reset.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""
+Reset a PostgreSQL StatefulSet (Kubernetes) to a clean state.
+
+What it does:
+1) Scale the StatefulSet down to 0 (graceful stop).
+2) Delete the data PVC for a given pod ordinal (wipes data).
+3) (Optional) Delete the backing PV if reclaimPolicy=Retain or you want to force-remove it.
+4) Scale the StatefulSet back up to 1 (or a target) and wait for readiness.
+
+Defaults match your example:
+  namespace: nc_db
+  statefulset: postgres
+  volumeClaimTemplates name: data
+  ordinal: 0
+  scale-up replicas: 1
+
+Usage examples:
+  python k8s/scripts/full_pod_reset.py
+  python k8s/scripts/full_pod_reset.py --namespace nc_db --statefulset postgres --vct data --ordinal 0 --replicas 1 --delete-pv
+  KUBECTL_BIN="microk8s.kubectl" python k8s/scripts/full_pod_reset.py
+
+Requires:
+  - kubectl (or microk8s.kubectl) in PATH
+  - current kube-context pointing at the correct cluster/namespace
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+import time
+from typing import Optional, Tuple
+
+
+def run_cmd(cmd: list[str], check: bool = True, capture: bool = True) -> Tuple[int, str]:
+    """Run a shell command. Returns (exit_code, stdout). Raises on error if check=True."""
+    proc = subprocess.run(
+        cmd,
+        check=False,
+        text=True,
+        stdout=subprocess.PIPE if capture else None,
+        stderr=subprocess.STDOUT,
+    )
+    if check and proc.returncode != 0:
+        raise RuntimeError(f"Command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stdout}")
+    return proc.returncode, (proc.stdout or "")
+
+
+def wait_for_pod_gone(kubectl: str, ns: str, pod: str, timeout_s: int = 180) -> None:
+    """Wait until the specific pod no longer exists."""
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        rc, _ = run_cmd([kubectl, "-n", ns, "get", "pod", pod], check=False)
+        if rc != 0:
+            return
+        time.sleep(2)
+    raise TimeoutError(f"Timed out waiting for pod/{pod} to be deleted")
+
+
+def wait_for_rollout_ready(kubectl: str, ns: str, sts: str, timeout_s: int = 300) -> None:
+    """Wait for StatefulSet rollout to complete."""
+    _, out = run_cmd([kubectl, "-n", ns, "rollout", "status", f"statefulset/{sts}", f"--timeout={timeout_s}s"])
+    print(out.strip())
+
+
+def get_pv_name_for_pvc(kubectl: str, ns: str, pvc: str) -> Optional[str]:
+    """Return the PV name bound to a PVC (or None if not bound/not found)."""
+    rc, out = run_cmd(
+        [kubectl, "-n", ns, "get", "pvc", pvc, "-o", "jsonpath={.spec.volumeName}"],
+        check=False,
+    )
+    pv = out.strip()
+    if rc != 0 or not pv:
+        return None
+    return pv
+
+
+def get_pv_reclaim_policy(kubectl: str, pv: str) -> Optional[str]:
+    """Return reclaim policy for a PV (e.g., Retain/Delete/Recycle), or None if not found."""
+    rc, out = run_cmd(
+        [kubectl, "get", "pv", pv, "-o", "jsonpath={.spec.persistentVolumeReclaimPolicy}"],
+        check=False,
+    )
+    pol = out.strip()
+    if rc != 0 or not pol:
+        return None
+    return pol
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Factory-reset a PostgreSQL StatefulSet (wipe data PVC and restart).")
+    parser.add_argument("--namespace", "-n", default="nc_db", help="Kubernetes namespace")
+    parser.add_argument("--statefulset", "--sts", default="postgres", dest="statefulset", help="StatefulSet name")
+    parser.add_argument("--vct", default="data", help="volumeClaimTemplates name (e.g., 'data')")
+    parser.add_argument("--ordinal", type=int, default=0, help="Pod ordinal to reset (e.g., 0 for <sts>-0)")
+    parser.add_argument("--replicas", type=int, default=1, help="Replicas to scale up after wipe")
+    parser.add_argument("--delete-pv", action="store_true", help="Also delete the bound PV (useful if Retain)")
+    parser.add_argument("--kubectl", default=os.environ.get("KUBECTL_BIN", "kubectl"), help="kubectl binary to use")
+    parser.add_argument("--yes", "-y", action="store_true", help="Do not prompt for confirmation")
+    args = parser.parse_args()
+
+    ns = args.namespace
+    sts = args.statefulset
+    vct = args.vct
+    ordinal = args.ordinal
+    replicas = args.replicas
+    kubectl = args.kubectl
+
+    pod_name = f"{sts}-{ordinal}"
+    pvc_name = f"{vct}-{pod_name}"
+
+    print(f"Namespace: {ns}")
+    print(f"StatefulSet: {sts}")
+    print(f"Target pod ordinal: {ordinal} -> pod/{pod_name}")
+    print(f"PVC to delete: {pvc_name}")
+    print(f"Scale up replicas after wipe: {replicas}")
+    print(f"Kubectl: {kubectl}")
+
+    if not args.yes:
+        resp = input("Proceed with FULL RESET (this will DELETE data on the PVC)? [y/N]: ").strip().lower()
+        if resp not in ("y", "yes"):
+            print("Aborted.")
+            sys.exit(1)
+
+    # 1) Scale down to 0
+    print("\n[1/5] Scaling StatefulSet down to 0 replicas...")
+    run_cmd([kubectl, "-n", ns, "scale", f"statefulset/{sts}", "--replicas=0"])
+    print("Waiting for pod to terminate...")
+    try:
+        wait_for_pod_gone(kubectl, ns, pod_name, timeout_s=180)
+    except TimeoutError as e:
+        print(f"Warning: {e}. Continuing…")
+
+    # Resolve PV name BEFORE deleting the PVC
+    pv_name = get_pv_name_for_pvc(kubectl, ns, pvc_name)
+    if pv_name:
+        print(f"PVC {pvc_name} is bound to PV {pv_name}")
+        reclaim = get_pv_reclaim_policy(kubectl, pv_name) or "Unknown"
+        print(f"PV reclaimPolicy: {reclaim}")
+    else:
+        print(f"Could not resolve a PV for PVC {pvc_name} (it may not exist or be Pending).")
+
+    # 2) Delete the PVC (the wipe)
+    print(f"\n[2/5] Deleting PVC {pvc_name}...")
+    run_cmd([kubectl, "-n", ns, "delete", "pvc", pvc_name, "--ignore-not-found=true"])
+
+    # 3) Optionally delete the PV
+    if args.delete_pv and pv_name:
+        print(f"\n[3/5] Deleting PV {pv_name} (because --delete-pv was set)...")
+        run_cmd([kubectl, "delete", "pv", pv_name])
+    else:
+        print("\n[3/5] Skipping PV deletion.")
+
+    # 4) Scale up
+    print(f"\n[4/5] Scaling StatefulSet up to {replicas} replicas...")
+    run_cmd([kubectl, "-n", ns, "scale", f"statefulset/{sts}", f"--replicas={replicas}"])
+
+    # 5) Wait for rollout ready
+    print("\n[5/5] Waiting for rollout to complete…")
+    wait_for_rollout_ready(kubectl, ns, sts, timeout_s=600)
+
+    print("\n✅ Reset complete. A new PVC will have been created automatically by the StatefulSet.")
+    print("Tip: watch the pod come up with:")
+    print(f"  {kubectl} -n {ns} get pods -w")
+    print("To view logs:")
+    print(f"  {kubectl} -n {ns} logs statefulset/{sts} --tail=100 -f")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print("\nInterrupted.")
+        sys.exit(130)
+    except Exception as exc:
+        print(f"\n❌ Error: {exc}")
+        sys.exit(1)