dehelpers 0.1.0__py3-none-any.whl

dehelpers/__init__.py

@@ -0,0 +1,30 @@
+"""dehelpers: Lightweight utilities for data engineering pipelines."""
+
+from dehelpers.api import NextLinkPagination, ResilientClient, RetryPolicy
+from dehelpers.db import DatabaseManager
+from dehelpers.exceptions import (
+    DatabaseError,
+    DPHError,
+    PaginationError,
+    RetryError,
+)
+from dehelpers.logger import LogContext, get_logger
+
+__all__ = [
+    # API
+    "ResilientClient",
+    "RetryPolicy",
+    "NextLinkPagination",
+    # Database
+    "DatabaseManager",
+    # Logger
+    "get_logger",
+    "LogContext",
+    # Exceptions
+    "DPHError",
+    "RetryError",
+    "PaginationError",
+    "DatabaseError",
+]
+
+__version__ = "0.1.0"

dehelpers/_redact.py

@@ -0,0 +1,122 @@
+"""Shared redaction utilities.
+
+Private module — not part of the public API.  Used by the logger,
+API client, and database manager to strip sensitive values before
+they reach log output or ``__repr__`` strings.
+"""
+
+from __future__ import annotations
+
+import copy
+from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
+
+DEFAULT_SENSITIVE_KEYS: frozenset[str] = frozenset(
+    {
+        "password",
+        "secret",
+        "token",
+        "api_key",
+        "authorization",
+        "dsn",
+        "connection_string",
+        "credential",
+        "passphrase",
+        "private_key",
+        "client_secret",
+    }
+)
+
+REDACTED = "***REDACTED***"
+
+
+def _is_sensitive(key: str, sensitive_keys: frozenset[str]) -> bool:
+    """Return True if *key* contains any sensitive substring (case-insensitive)."""
+    key_lower = key.lower()
+    return any(s in key_lower for s in sensitive_keys)
+
+
+def redact_dict(
+    d: dict,
+    sensitive_keys: frozenset[str] | None = None,
+    extra_sensitive_keys: frozenset[str] | None = None,
+) -> dict:
+    """Deep-clone *d* and replace values whose keys match the sensitive set.
+
+    Matching is **case-insensitive substring** — e.g. a key named
+    ``db_password`` matches ``password``.
+
+    Parameters
+    ----------
+    d:
+        The dictionary to redact.  Not mutated.
+    sensitive_keys:
+        Override the full sensitive-key set.  When ``None``, uses
+        :data:`DEFAULT_SENSITIVE_KEYS`.
+    extra_sensitive_keys:
+        Additional keys to treat as sensitive, merged with *sensitive_keys*.
+
+    Returns
+    -------
+    dict
+        A deep copy of *d* with sensitive values replaced by
+        ``'***REDACTED***'``.
+    """
+    keys = sensitive_keys if sensitive_keys is not None else DEFAULT_SENSITIVE_KEYS
+    if extra_sensitive_keys:
+        keys = keys | extra_sensitive_keys
+
+    return _redact_recursive(d, keys)
+
+
+def _redact_recursive(obj: object, sensitive_keys: frozenset[str]) -> object:
+    """Recursively redact sensitive keys in nested structures."""
+    if isinstance(obj, dict):
+        result: dict = {}
+        for k, v in obj.items():
+            if isinstance(k, str) and _is_sensitive(k, sensitive_keys):
+                result[k] = REDACTED
+            else:
+                result[k] = _redact_recursive(v, sensitive_keys)
+        return result
+    if isinstance(obj, (list, tuple)):
+        redacted = [_redact_recursive(item, sensitive_keys) for item in obj]
+        return type(obj)(redacted)
+    return copy.deepcopy(obj) if isinstance(obj, (set, frozenset)) else obj
+
+
+def redact_url(
+    url: str,
+    sensitive_keys: frozenset[str] | None = None,
+) -> str:
+    """Return *url* with query-parameter values redacted for sensitive keys.
+
+    Parameters
+    ----------
+    url:
+        The URL to redact.
+    sensitive_keys:
+        Override the full sensitive-key set.  When ``None``, uses
+        :data:`DEFAULT_SENSITIVE_KEYS`.
+
+    Returns
+    -------
+    str
+        The URL with matching query-parameter values replaced by
+        ``'***REDACTED***'``.
+    """
+    keys = sensitive_keys if sensitive_keys is not None else DEFAULT_SENSITIVE_KEYS
+
+    parts = urlsplit(url)
+    if not parts.query:
+        return url
+
+    params = parse_qs(parts.query, keep_blank_values=True)
+    redacted_params: dict[str, list[str]] = {}
+    for k, values in params.items():
+        if _is_sensitive(k, keys):
+            redacted_params[k] = [REDACTED] * len(values)
+        else:
+            redacted_params[k] = values
+
+    new_query = urlencode(redacted_params, doseq=True, quote_via=lambda s, safe="", encoding=None, errors=None: s)
+    return urlunsplit(parts._replace(query=new_query))

dehelpers/api.py

@@ -0,0 +1,380 @@
+"""Resilient HTTP client with bounded retries, exponential backoff,
+jitter, total-timeout guard, and next-link pagination.
+
+Usage::
+
+    from dehelpers import ResilientClient, RetryPolicy
+
+    client = ResilientClient()
+    resp = client.get("https://api.example.com/data")
+
+    # Paginate through all items
+    for item in client.paginate("https://api.example.com/items"):
+        process(item)
+"""
+
+from __future__ import annotations
+
+import logging
+import random
+import time
+from dataclasses import dataclass, field
+from typing import Any, Iterator
+
+import requests
+
+from dehelpers._redact import redact_url
+from dehelpers.exceptions import PaginationError, RetryError
+
+__all__ = ["RetryPolicy", "NextLinkPagination", "ResilientClient"]
+
+# HTTP methods considered idempotent and therefore safe to retry.
+_IDEMPOTENT_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
+
+
+# ---------------------------------------------------------------------------
+# Retry Policy
+# ---------------------------------------------------------------------------
+@dataclass(frozen=True)
+class RetryPolicy:
+    """Configuration for retry behaviour.
+
+    Attributes
+    ----------
+    max_retries:
+        Maximum number of *retries* (not total attempts).  Total attempts
+        = ``max_retries + 1``.
+    backoff_base:
+        Base delay in seconds for exponential backoff.
+    backoff_max:
+        Maximum delay cap in seconds.
+    jitter:
+        If ``True``, adds random jitter to the delay to prevent
+        thundering-herd effects.
+    total_timeout:
+        Wall-clock cap in seconds measured from the **start of the first
+        attempt**.  Retries abort if this is exceeded.
+    retryable_statuses:
+        HTTP status codes that trigger a retry.
+    retry_non_idempotent:
+        If ``True``, retries POST/PUT/DELETE.  Default is ``False``
+        (only idempotent methods are retried).
+    connect_timeout:
+        Per-request TCP connect timeout in seconds.
+    read_timeout:
+        Per-request read timeout in seconds.
+    """
+
+    max_retries: int = 3
+    backoff_base: float = 1.0
+    backoff_max: float = 30.0
+    jitter: bool = True
+    total_timeout: float = 120.0
+    retryable_statuses: frozenset[int] = field(
+        default_factory=lambda: frozenset({429, 500, 502, 503, 504})
+    )
+    retry_non_idempotent: bool = False
+    connect_timeout: float = 5.0
+    read_timeout: float = 30.0
+
+
+# ---------------------------------------------------------------------------
+# Pagination
+# ---------------------------------------------------------------------------
+class NextLinkPagination:
+    """Follows a ``next`` URL key in the JSON response.
+
+    Parameters
+    ----------
+    next_key:
+        Key in the JSON response that contains the next page URL.
+    results_key:
+        Key in the JSON response that contains the list of items.
+    max_pages:
+        Safety limit on the number of pages to fetch.
+    """
+
+    def __init__(
+        self,
+        next_key: str = "next",
+        results_key: str = "results",
+        max_pages: int = 100,
+    ) -> None:
+        self.next_key = next_key
+        self.results_key = results_key
+        self.max_pages = max_pages
+
+
+# ---------------------------------------------------------------------------
+# Client
+# ---------------------------------------------------------------------------
+class ResilientClient:
+    """HTTP client with automatic retries, backoff, and pagination.
+
+    Parameters
+    ----------
+    retry_policy:
+        Retry configuration.  Uses sensible defaults when ``None``.
+    logger:
+        Logger instance.  A default JSON logger is created when ``None``.
+    """
+
+    def __init__(
+        self,
+        retry_policy: RetryPolicy | None = None,
+        logger: logging.Logger | None = None,
+    ) -> None:
+        self._policy = retry_policy or RetryPolicy()
+        self._log = logger or logging.getLogger(__name__)
+        self._session = requests.Session()
+
+    # -- Public helpers -----------------------------------------------------
+
+    def get(self, url: str, **kwargs: Any) -> requests.Response:
+        """Send a GET request with retry protection."""
+        return self.request("GET", url, **kwargs)
+
+    def post(self, url: str, **kwargs: Any) -> requests.Response:
+        """Send a POST request with retry protection."""
+        return self.request("POST", url, **kwargs)
+
+    def put(self, url: str, **kwargs: Any) -> requests.Response:
+        """Send a PUT request with retry protection."""
+        return self.request("PUT", url, **kwargs)
+
+    def delete(self, url: str, **kwargs: Any) -> requests.Response:
+        """Send a DELETE request with retry protection."""
+        return self.request("DELETE", url, **kwargs)
+
+    # -- Core request -------------------------------------------------------
+
+    def request(self, method: str, url: str, **kwargs: Any) -> requests.Response:
+        """Send an HTTP request with bounded retries and backoff.
+
+        Raises
+        ------
+        RetryError
+            When all retry attempts are exhausted or the total timeout
+            is exceeded.  The original exception is preserved as
+            ``__cause__``.
+        requests.HTTPError
+            On non-retryable HTTP errors (e.g. 400, 401, 403, 404).
+        """
+        policy = self._policy
+        method_upper = method.upper()
+        can_retry = (
+            method_upper in _IDEMPOTENT_METHODS or policy.retry_non_idempotent
+        )
+
+        timeout_tuple = kwargs.pop(
+            "timeout", (policy.connect_timeout, policy.read_timeout)
+        )
+
+        safe_url = redact_url(url)
+        start = time.monotonic()
+        last_exception: Exception | None = None
+        last_status: int | None = None
+
+        for attempt in range(policy.max_retries + 1):
+            # Total-timeout guard.
+            elapsed = time.monotonic() - start
+            if attempt > 0 and elapsed >= policy.total_timeout:
+                raise RetryError(
+                    f"Total timeout ({policy.total_timeout}s) exceeded after "
+                    f"{attempt} attempt(s) for {method_upper} {safe_url}",
+                    last_status=last_status,
+                    attempts=attempt,
+                ) from last_exception
+
+            try:
+                resp = self._session.request(
+                    method_upper, url, timeout=timeout_tuple, **kwargs
+                )
+                last_status = resp.status_code
+
+                self._log.info(
+                    "HTTP %s %s -> %d (attempt %d/%d)",
+                    method_upper,
+                    safe_url,
+                    resp.status_code,
+                    attempt + 1,
+                    policy.max_retries + 1,
+                )
+
+                # Success — return immediately.
+                if resp.status_code < 400:
+                    return resp
+
+                # Retryable status?
+                if (
+                    can_retry
+                    and resp.status_code in policy.retryable_statuses
+                    and attempt < policy.max_retries
+                ):
+                    delay = self._compute_delay(attempt, resp)
+                    self._log.warning(
+                        "Retryable %d for %s %s — sleeping %.2fs",
+                        resp.status_code,
+                        method_upper,
+                        safe_url,
+                        delay,
+                    )
+                    time.sleep(delay)
+                    continue
+
+                 # Non-retryable HTTP error — raise immediately.
+                resp.raise_for_status()
+
+            except requests.exceptions.HTTPError as exc:
+                # If the status is in retryable_statuses but we ran out of retries,
+                # wrap it in a RetryError.  Otherwise (like 400, 401, 404), raise it immediately.
+                if last_status in policy.retryable_statuses:
+                    raise RetryError(
+                        f"All {policy.max_retries + 1} attempts exhausted for "
+                        f"{method_upper} {safe_url} (last status: {last_status})",
+                        last_status=last_status,
+                        attempts=attempt + 1,
+                    ) from exc
+                raise exc
+            except requests.RequestException as exc:
+                last_exception = exc
+                if (
+                    can_retry
+                    and attempt < policy.max_retries
+                ):
+                    delay = self._compute_delay(attempt)
+                    self._log.warning(
+                        "Connection error for %s %s (attempt %d/%d) — %s — "
+                        "sleeping %.2fs",
+                        method_upper,
+                        safe_url,
+                        attempt + 1,
+                        policy.max_retries + 1,
+                        type(exc).__name__,
+                        delay,
+                    )
+                    time.sleep(delay)
+                    continue
+                raise RetryError(
+                    f"Request failed after {attempt + 1} attempt(s) for "
+                    f"{method_upper} {safe_url}: {exc}",
+                    last_status=last_status,
+                    attempts=attempt + 1,
+                ) from exc
+
+        # All retries exhausted with an HTTP error status.
+        raise RetryError(
+            f"All {policy.max_retries + 1} attempts exhausted for "
+            f"{method_upper} {safe_url} (last status: {last_status})",
+            last_status=last_status,
+            attempts=policy.max_retries + 1,
+        ) from last_exception
+
+    # -- Pagination ---------------------------------------------------------
+
+    def paginate(
+        self,
+        url: str,
+        pagination: NextLinkPagination | None = None,
+        **kwargs: Any,
+    ) -> Iterator[dict]:
+        """Yield individual items across paginated responses.
+
+        Parameters
+        ----------
+        url:
+            Initial page URL.
+        pagination:
+            Pagination strategy.  Defaults to :class:`NextLinkPagination`.
+        **kwargs:
+            Extra keyword arguments forwarded to each GET request.
+
+        Yields
+        ------
+        dict
+            Individual items from each page.
+
+        Raises
+        ------
+        PaginationError
+            On any failure.  ``PaginationError.collected_items``
+            contains items fetched before the failure.
+        """
+        pag = pagination or NextLinkPagination()
+        collected: list[dict] = []
+        current_url: str | None = url
+
+        for page_num in range(1, pag.max_pages + 1):
+            if current_url is None:
+                return
+
+            try:
+                resp = self.get(current_url, **kwargs)
+                resp.raise_for_status()
+                data = resp.json()
+            except Exception as exc:
+                raise PaginationError(
+                    f"Failed on page {page_num}: {exc}",
+                    collected_items=collected,
+                    cause=exc,
+                ) from exc
+
+            items = data.get(pag.results_key, [])
+            if not items:
+                return
+
+            collected.extend(items)
+            yield from items
+
+            # Validate the 'next' field.
+            next_val = data.get(pag.next_key)
+            if next_val is None:
+                return
+            if not isinstance(next_val, str):
+                raise PaginationError(
+                    f"Expected '{pag.next_key}' to be a string URL or None, "
+                    f"got {type(next_val).__name__}: {next_val!r}",
+                    collected_items=collected,
+                )
+            current_url = next_val
+
+        self._log.warning("Reached max_pages limit (%d)", pag.max_pages)
+
+    # -- Internal -----------------------------------------------------------
+
+    def _compute_delay(
+        self,
+        attempt: int,
+        response: requests.Response | None = None,
+    ) -> float:
+        """Calculate backoff delay for the given attempt.
+
+        Respects ``Retry-After`` header on 429 responses when available.
+        """
+        policy = self._policy
+
+        # Honour Retry-After header if present (429 responses).
+        if response is not None and response.status_code == 429:
+            retry_after = response.headers.get("Retry-After")
+            if retry_after is not None:
+                try:
+                    return max(0.0, float(retry_after))
+                except (ValueError, TypeError):
+                    pass  # Fall through to normal backoff.
+
+        delay = min(policy.backoff_base * (2**attempt), policy.backoff_max)
+        if policy.jitter:
+            delay += random.uniform(0, delay * 0.25)  # noqa: S311
+        return delay
+
+    # -- Cleanup ------------------------------------------------------------
+
+    def close(self) -> None:
+        """Close the underlying requests session."""
+        self._session.close()
+
+    def __enter__(self) -> ResilientClient:
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.close()

dehelpers/db.py

@@ -0,0 +1,243 @@
+"""PostgreSQL-first database helper with safe connection pooling.
+
+Built on SQLAlchemy 2.0 with context-managed sessions, pre-ping
+health checks, connection recycling, and optional Pandas DataFrame
+output.
+
+Usage::
+
+    from dehelpers import DatabaseManager
+
+    with DatabaseManager() as db:          # reads DATABASE_URL env var
+        rows = db.execute("SELECT * FROM users WHERE active = :active",
+                          {"active": True})
+        df = db.to_dataframe("SELECT * FROM sales")
+
+.. warning::
+
+    If using in forked environments (Airflow, multiprocessing), create
+    the ``DatabaseManager`` **inside each worker process** or call
+    :meth:`dispose` before forking.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+from sqlalchemy import Row, create_engine, text
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session, sessionmaker
+
+from dehelpers._redact import redact_url
+from dehelpers.exceptions import DatabaseError
+
+__all__ = ["DatabaseManager"]
+
+
+class DatabaseManager:
+    """PostgreSQL connection manager with safe pooling defaults.
+
+    Parameters
+    ----------
+    dsn:
+        SQLAlchemy connection URL.  Falls back to the ``DATABASE_URL``
+        environment variable when ``None``.
+    pool_size:
+        Number of persistent connections in the pool.
+    max_overflow:
+        Maximum additional connections beyond *pool_size*.
+    pool_recycle:
+        Seconds before a connection is recycled (replaced).
+    pool_pre_ping:
+        If ``True``, issues a lightweight ``SELECT 1`` before checking
+        out a connection to verify it is still alive.
+    pool_timeout:
+        Seconds to wait for a connection from the pool before raising
+        :class:`~dehelpers.exceptions.DatabaseError`.
+    """
+
+    def __init__(
+        self,
+        dsn: str | None = None,
+        *,
+        pool_size: int = 5,
+        max_overflow: int = 2,
+        pool_recycle: int = 1800,
+        pool_pre_ping: bool = True,
+        pool_timeout: int = 30,
+    ) -> None:
+        resolved_dsn = dsn or os.environ.get("DATABASE_URL")
+        if not resolved_dsn:
+            raise DatabaseError(
+                "No DSN provided and DATABASE_URL environment variable is not set."
+            )
+        self._dsn = resolved_dsn
+
+        # SQLite uses SingletonThreadPool which doesn't support pool_size,
+        # max_overflow, or pool_timeout.  Only pass those for real backends.
+        engine_kwargs: dict[str, object] = {
+            "pool_recycle": pool_recycle,
+            "pool_pre_ping": pool_pre_ping,
+        }
+        if not resolved_dsn.startswith("sqlite"):
+            engine_kwargs.update(
+                pool_size=pool_size,
+                max_overflow=max_overflow,
+                pool_timeout=pool_timeout,
+            )
+        self._engine = create_engine(resolved_dsn, **engine_kwargs)
+        self._session_factory = sessionmaker(bind=self._engine)
+
+    # -- Context manager ----------------------------------------------------
+
+    def __enter__(self) -> DatabaseManager:
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.dispose()
+
+    # -- Session management -------------------------------------------------
+
+    def session(self) -> _SessionContext:
+        """Return a context manager that yields a SQLAlchemy ``Session``.
+
+        Auto-commits on clean exit and auto-rolls-back on exception.
+
+        Example::
+
+            with db.session() as session:
+                session.execute(text("INSERT INTO logs ..."))
+        """
+        return _SessionContext(self._session_factory)
+
+    # -- Query shortcuts ----------------------------------------------------
+
+    def execute(
+        self,
+        sql: str,
+        params: dict[str, Any] | None = None,
+    ) -> list[Row]:
+        """Execute *sql* and return all rows as a ``list[Row]``.
+
+        The connection is returned to the pool immediately after.
+
+        Parameters
+        ----------
+        sql:
+            SQL string (use ``:param`` style placeholders).
+        params:
+            Bind parameters.
+
+        Returns
+        -------
+        list[Row]
+            Rows from the query result.  Each :class:`~sqlalchemy.engine.Row`
+            supports both index and attribute access.
+        """
+        try:
+            with self._engine.connect() as conn:
+                result = conn.execute(text(sql), params or {})
+                rows = list(result.fetchall())
+                conn.commit()
+                return rows
+        except SQLAlchemyError as exc:
+            raise DatabaseError(f"Query execution failed: {exc}") from exc
+
+    def fetch_one(
+        self,
+        sql: str,
+        params: dict[str, Any] | None = None,
+    ) -> Row | None:
+        """Execute *sql* and return the first row, or ``None``.
+
+        Parameters
+        ----------
+        sql:
+            SQL string.
+        params:
+            Bind parameters.
+        """
+        try:
+            with self._engine.connect() as conn:
+                result = conn.execute(text(sql), params or {})
+                row = result.fetchone()
+                conn.commit()
+                return row
+        except SQLAlchemyError as exc:
+            raise DatabaseError(f"Query execution failed: {exc}") from exc
+
+    def to_dataframe(
+        self,
+        sql: str,
+        params: dict[str, Any] | None = None,
+    ) -> Any:  # -> pd.DataFrame (lazy import)
+        """Execute *sql* and return the result as a Pandas DataFrame.
+
+        Requires the ``[dataframe]`` extra::
+
+            pip install dehelpers[dataframe]
+
+        Raises
+        ------
+        ImportError
+            If ``pandas`` is not installed.
+        DatabaseError
+            On query failure.
+        """
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ImportError(
+                "pandas is required for to_dataframe(). "
+                "Install it with: pip install dehelpers[dataframe]"
+            ) from None
+
+        try:
+            with self._engine.connect() as conn:
+                return pd.read_sql(text(sql), conn, params=params or {})
+        except SQLAlchemyError as exc:
+            raise DatabaseError(
+                f"DataFrame query failed: {exc}"
+            ) from exc
+
+    # -- Cleanup ------------------------------------------------------------
+
+    def dispose(self) -> None:
+        """Dispose the engine and close all pooled connections."""
+        self._engine.dispose()
+
+    # -- Repr (redacted) ----------------------------------------------------
+
+    def __repr__(self) -> str:
+        safe = redact_url(self._dsn)
+        return f"DatabaseManager(dsn={safe!r})"
+
+
+# ---------------------------------------------------------------------------
+# Session context manager
+# ---------------------------------------------------------------------------
+class _SessionContext:
+    """Internal context manager wrapping a SQLAlchemy Session.
+
+    Commits on clean exit, rolls back on exception.
+    """
+
+    def __init__(self, factory: sessionmaker) -> None:
+        self._factory = factory
+        self._session: Session | None = None
+
+    def __enter__(self) -> Session:
+        self._session = self._factory()
+        return self._session
+
+    def __exit__(self, exc_type: type | None, *_: object) -> None:
+        if self._session is None:
+            return
+        try:
+            if exc_type is None:
+                self._session.commit()
+            else:
+                self._session.rollback()
+        finally:
+            self._session.close()

dehelpers/exceptions.py

@@ -0,0 +1,66 @@
+"""Custom exceptions for dehelpers.
+
+Centralised exception hierarchy so every module raises from here
+instead of scattering ad-hoc exceptions.
+"""
+
+__all__ = ["DPHError", "RetryError", "PaginationError", "DatabaseError"]
+
+
+class DPHError(Exception):
+    """Base exception for dehelpers."""
+
+
+class RetryError(DPHError):
+    """Raised when all retry attempts are exhausted or total timeout exceeded.
+
+    The original exception (connection error, timeout, etc.) is always
+    preserved as ``__cause__`` via ``raise RetryError(...) from original``.
+
+    Attributes:
+        last_status: HTTP status code of the last attempt, or ``None``
+            if the failure was a connection-level error.
+        attempts: Total number of attempts made (including the first).
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        last_status: int | None = None,
+        attempts: int = 0,
+    ) -> None:
+        super().__init__(message)
+        self.last_status = last_status
+        self.attempts = attempts
+
+
+class PaginationError(DPHError):
+    """Raised on pagination failure.
+
+    Carries items collected before the failure so callers can decide
+    whether to use partial results.
+
+    Attributes:
+        collected_items: Items successfully fetched before the error.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        collected_items: list[dict] | None = None,
+        cause: Exception | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.collected_items = collected_items or []
+        if cause is not None:
+            self.__cause__ = cause
+
+
+class DatabaseError(DPHError):
+    """Raised on database operation failures.
+
+    Wraps SQLAlchemy or driver-level exceptions while keeping the
+    original available via ``__cause__``.
+    """

dehelpers/logger.py

@@ -0,0 +1,217 @@
+"""Structured JSON logger with automatic secret redaction.
+
+Built entirely on the stdlib :mod:`logging` module — no third-party
+dependencies.  Every log record is emitted as a single JSON line with
+a consistent schema, making it ready for containerised environments
+like Airflow, AWS ECS, or Google Cloud Run.
+
+Usage::
+
+    from dehelpers import get_logger, LogContext
+
+    log = get_logger("my_etl", job_id="daily-sales")
+    log.info("Starting extraction", extra={"source": "api"})
+
+    with LogContext(request_id="abc-123"):
+        log.info("Fetched page", extra={"page": 1})
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+import traceback
+from contextvars import ContextVar
+from datetime import datetime, timezone
+from typing import Any
+
+from dehelpers._redact import redact_dict
+
+__all__ = ["get_logger", "LogContext"]
+
+# ---------------------------------------------------------------------------
+# Context variables for cross-cutting fields
+# ---------------------------------------------------------------------------
+_ctx_job_id: ContextVar[str | None] = ContextVar("job_id", default=None)
+_ctx_request_id: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+
+class LogContext:
+    """Context manager that injects ``job_id`` and/or ``request_id``
+    into every log record emitted within its scope.
+
+    Example::
+
+        with LogContext(job_id="etl-run-42", request_id="req-abc"):
+            logger.info("Processing")
+    """
+
+    def __init__(
+        self,
+        *,
+        job_id: str | None = None,
+        request_id: str | None = None,
+    ) -> None:
+        self._job_id = job_id
+        self._request_id = request_id
+        self._tokens: list = []
+
+    def __enter__(self) -> LogContext:
+        if self._job_id is not None:
+            self._tokens.append(_ctx_job_id.set(self._job_id))
+        if self._request_id is not None:
+            self._tokens.append(_ctx_request_id.set(self._request_id))
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        for token in reversed(self._tokens):
+            token.var.reset(token)
+        self._tokens.clear()
+
+
+# ---------------------------------------------------------------------------
+# JSON Formatter
+# ---------------------------------------------------------------------------
+class _JSONFormatter(logging.Formatter):
+    """Formats log records as single-line JSON with redaction.
+
+    Schema (every record)::
+
+        {
+          "timestamp": "2026-07-02T11:43:50.123456Z",
+          "level": "INFO",
+          "message": "Fetched 200 rows",
+          "module": "db",
+          "function": "execute_query",
+          "job_id": "etl-daily-sales",
+          "request_id": null,
+          "error": null
+        }
+    """
+
+    # Fields injected by logging internals that we don't want in output.
+    _INTERNAL_KEYS = frozenset(
+        {
+            "name",
+            "msg",
+            "args",
+            "created",
+            "relativeCreated",
+            "exc_info",
+            "exc_text",
+            "stack_info",
+            "lineno",
+            "funcName",
+            "pathname",
+            "filename",
+            "levelno",
+            "levelname",
+            "module",
+            "msecs",
+            "process",
+            "processName",
+            "thread",
+            "threadName",
+            "taskName",
+            "message",
+        }
+    )
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Return a single-line JSON string for *record*."""
+        try:
+            return self._safe_format(record)
+        except Exception:
+            # Recursion / serialization guard: fallback to plain text.
+            return json.dumps(
+                {
+                    "timestamp": datetime.now(timezone.utc).isoformat(),
+                    "level": "ERROR",
+                    "message": f"[FORMATTER ERROR] {record.getMessage()}",
+                    "module": getattr(record, "module", "unknown"),
+                    "function": getattr(record, "funcName", "unknown"),
+                    "_formatter_error": True,
+                }
+            )
+
+    def _safe_format(self, record: logging.LogRecord) -> str:
+        # Build the structured payload.
+        payload: dict[str, Any] = {
+            "timestamp": datetime.fromtimestamp(
+                record.created, tz=timezone.utc
+            ).isoformat(),
+            "level": record.levelname,
+            "message": record.getMessage(),
+            "module": record.module,
+            "function": record.funcName,
+            "job_id": _ctx_job_id.get(),
+            "request_id": _ctx_request_id.get(),
+            "error": None,
+        }
+
+        # Merge user-supplied extra fields (redacted).
+        extras: dict[str, Any] = {
+            k: v
+            for k, v in record.__dict__.items()
+            if k not in self._INTERNAL_KEYS
+        }
+        if extras:
+            payload.update(redact_dict(extras))
+
+        # Serialise exception info if present.
+        if record.exc_info and record.exc_info[0] is not None:
+            exc_type, exc_value, exc_tb = record.exc_info
+            payload["error"] = {
+                "type": exc_type.__name__ if exc_type else "Unknown",
+                "message": str(exc_value),
+                "traceback": traceback.format_exception(
+                    exc_type, exc_value, exc_tb
+                )[-3:],  # last 3 frames to keep logs concise
+            }
+
+        return json.dumps(payload, default=str)
+
+
+# ---------------------------------------------------------------------------
+# Public factory
+# ---------------------------------------------------------------------------
+def get_logger(
+    name: str,
+    *,
+    job_id: str | None = None,
+    level: int = logging.INFO,
+) -> logging.Logger:
+    """Return a stdlib :class:`~logging.Logger` with JSON formatting.
+
+    Parameters
+    ----------
+    name:
+        Logger name (typically the module or pipeline name).
+    job_id:
+        Optional default job identifier injected into every record.
+        Can also be set/overridden at runtime via :class:`LogContext`.
+    level:
+        Logging level.  Defaults to ``INFO``.
+
+    Returns
+    -------
+    logging.Logger
+        A configured logger that writes JSON to *stderr*.
+    """
+    if job_id is not None:
+        _ctx_job_id.set(job_id)
+
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+
+    # Avoid duplicate handlers on repeated calls.
+    if not any(isinstance(h, logging.StreamHandler) for h in logger.handlers):
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setFormatter(_JSONFormatter())
+        logger.addHandler(handler)
+
+    # Don't propagate to root logger to avoid double output.
+    logger.propagate = False
+
+    return logger

dehelpers/py.typed

@@ -0,0 +1 @@
+

dehelpers-0.1.0.dist-info/METADATA

@@ -0,0 +1,279 @@
+Metadata-Version: 2.4
+Name: dehelpers
+Version: 0.1.0
+Summary: Lightweight utilities for data engineering pipelines: resilient HTTP, PostgreSQL helpers, and structured logging.
+Project-URL: Homepage, https://github.com/shard-c6/dehelpers
+Project-URL: Repository, https://github.com/shard-c6/dehelpers
+Author: Shardul Chogale
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: psycopg[binary]>=3.0
+Requires-Dist: requests>=2.28
+Requires-Dist: sqlalchemy>=2.0
+Provides-Extra: dataframe
+Requires-Dist: pandas>=2.0; extra == 'dataframe'
+Provides-Extra: dev
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest-postgresql; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: responses>=0.23; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# dehelpers
+
+Lightweight, production-hardened Python utilities for data engineering pipelines.
+
+**Resilient HTTP** · **PostgreSQL helpers** · **Structured JSON logging** — with automatic secret redaction, bounded retries, and safe connection pooling.
+
+---
+
+## Architecture & Flow
+
+```mermaid
+graph TD
+    subgraph External [External APIs & Services]
+        REST_API[REST API Source]
+    end
+
+    subgraph DPH [dehelpers Package]
+        direction TB
+        subgraph Client [Resilient Client]
+            RC[ResilientClient] --> |Configured by| RP[RetryPolicy]
+            RC --> |Iterates with| NLP[NextLinkPagination]
+            RC --> |Sanitizes query| RU[redact_url]
+        end
+
+        subgraph Logger [Structured Logger]
+            GL[get_logger] --> |Formats record| JF[JSONFormatter]
+            LogCtx[LogContext] --> |Context injection| CV[job_id / request_id]
+            JF --> |Deep-redacts secrets| RD[redact_dict]
+        end
+
+        subgraph Database [Database Manager]
+            DBM[DatabaseManager] --> |Yields sessions| SC[_SessionContext]
+            DBM --> |Manages pool| SQLA[SQLAlchemy Engine]
+            DBM --> |Lazy Load| DF[Pandas DataFrame]
+        end
+    end
+
+    subgraph Target [Storage / Logs]
+        PG[(PostgreSQL DB)]
+        Stderr[Stderr / Cloud Logs]
+    end
+
+    REST_API ==> |Inbound Data| RC
+    RC --> |Yields items / logs events| GL
+    GL ==> |JSON Output| Stderr
+    RC --> |Normalized data| DBM
+    DBM ==> |Pool connections| PG
+```
+
+---
+
+## Boundaries & Capabilities
+
+Here is exactly what this package **is** and what it **is not**:
+
+| Category / Layer | What this IS | What this IS NOT |
+|:---|:---|:---|
+| **API / HTTP** | A retry-protected wrapper around `requests.Session` with exponential backoff, jitter, and simple pagination. | An asynchronous network library (like `aiohttp` or `httpx`), fully-fledged HTTP client replacement, or GraphQL API wrapper. |
+| **Database** | A thread-safe connection manager for PostgreSQL with pooling configuration, automated transaction commits/rollbacks, and lazy DataFrame output. | An Object-Relational Mapper (ORM) (like SQLModel/SQLAlchemy ORM), schema migration engine (like Alembic), or database administration tool. |
+| **Logging** | A zero-dependency structured JSON formatter on top of standard `logging` with automatic deep secrets redaction. | A log routing system (like Fluentd/Logstash), file logger, metrics exporter, or complex log management server. |
+| **Execution Context** | Designed for batch execution environments like Airflow tasks, ETL scripts, and containerized Docker runtimes. | Suitable for high-throughput, low-latency, real-time web servers or async microservices. |
+
+---
+
+## Comparison with Standard Setup
+
+How this package compares to a standard DIY setup:
+
+| Feature / Criteria | Standard Setup (`requests` + `logging` + `psycopg`) | `dehelpers` |
+|:---|:---|:---|
+| **Secret Leakage Protection** | Manual / None. Secrets easily print to stdout or appear in exception tracebacks. | **Automatic & Deep Recursive:** Redacts predefined secrets from nested metadata, logs, and query parameters. |
+| **Retry & Jitter Strategy** | Manual loops or boilerplate `urllib3` retry configurations. | **Out-of-the-box resilience:** Exponential backoff with random jitter and clock-based `total_timeout` limit. |
+| **Pagination Handling** | Custom pagination loop logic required for every API endpoint. | **Next-link strategy Protocol:** Yields individual items transparently and safely with validation. |
+| **Connection Safety** | Connection leaks or transaction rollback failures if block managers are missed. | **Context-managed Session:** Engine-pooled with pre-ping checks, pool timeout, and auto-rollback. |
+| **Dependency Footprint** | Heavy setup if installing frameworks like Loguru, Structlog, or heavy database utilities. | **Ultra-lightweight:** Base dependencies are minimal. Pandas is entirely optional and lazy-loaded. |
+
+---
+
+## Roadmap & What's Next
+
+| Phase | Feature / Expansion | Target Use Case | Status |
+|:---|:---|:---|:---|
+| **v1.0** | Core Resilient HTTP, Postgres Pool, Redacted Logger | Personal ETL scripts & Airflow workflows | **Released** |
+| **v1.1** | Cursor-based Pagination (`CursorPagination`) | Handling APIs that use cursor-based cursors | *Planned* |
+| **v1.2** | Async Client Support (`AsyncResilientClient`) | High-throughput concurrent API extraction pipelines | *Planned* |
+| **v1.3** | Parquet / Arrow Ingestion Support | High-performance bulk column-based ingestion | *Planned* |
+| **v2.0** | Schema Validation Layer (`pydantic` integration) | Ingestion payload sanitization and schema contracts | *Conceptual* |
+
+---
+
+## Install
+
+```bash
+# Core (HTTP + DB + logging)
+pip install dehelpers
+
+# With Pandas DataFrame support
+pip install dehelpers[dataframe]
+
+# Development (tests)
+pip install dehelpers[dev,dataframe]
+```
+
+Requires Python ≥ 3.10.
+
+---
+
+## Quickstart
+
+### Resilient HTTP Client
+
+```python
+from dehelpers import ResilientClient, RetryPolicy
+
+# Custom policy: 5 retries, retry POST with opt-in
+policy = RetryPolicy(max_retries=5, retry_non_idempotent=True)
+client = ResilientClient(retry_policy=policy)
+
+resp = client.get("https://api.example.com/data")
+print(resp.json())
+
+# Paginate through all items
+for item in client.paginate("https://api.example.com/items"):
+    process(item)
+```
+
+### PostgreSQL Database Helper
+
+```python
+from dehelpers import DatabaseManager
+
+# Reads DATABASE_URL from environment by default
+with DatabaseManager() as db:
+    rows = db.execute(
+        "SELECT * FROM users WHERE active = :active",
+        {"active": True},
+    )
+    print(f"Found {len(rows)} active users")
+
+    # Optional: load into a Pandas DataFrame
+    df = db.to_dataframe("SELECT * FROM sales WHERE date > :d", {"d": "2026-01-01"})
+```
+
+### Structured JSON Logger
+
+```python
+from dehelpers import get_logger, LogContext
+
+log = get_logger("my_etl", job_id="daily-sales")
+
+with LogContext(request_id="req-abc"):
+    log.info("Fetched data", extra={"row_count": 500})
+    # Output: {"timestamp": "...", "level": "INFO", "message": "Fetched data",
+    #          "module": "...", "job_id": "daily-sales", "request_id": "req-abc",
+    #          "row_count": 500, "error": null}
+```
+
+---
+
+## Configuration
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `DATABASE_URL` (env var) | — | PostgreSQL connection string (fallback when `dsn` is not passed) |
+| `pool_size` | 5 | Persistent connections in the pool |
+| `max_overflow` | 2 | Extra connections beyond pool_size |
+| `pool_recycle` | 1800 | Seconds before connection recycling |
+| `pool_pre_ping` | True | Health-check connections before use |
+| `pool_timeout` | 30 | Seconds to wait for a pool connection |
+
+---
+
+## Security
+
+### Automatic Redaction
+
+The logger and API client automatically redact values for these keys in log output:
+
+`password`, `secret`, `token`, `api_key`, `authorization`, `dsn`, `connection_string`, `credential`, `passphrase`, `private_key`, `client_secret`
+
+Matching is **case-insensitive substring** — e.g. `db_password` matches `password`.
+
+You can extend the redaction list:
+
+```python
+from dehelpers._redact import redact_dict
+
+result = redact_dict(
+    {"my_custom_secret": "value"},
+    extra_sensitive_keys=frozenset({"my_custom_secret"}),
+)
+```
+
+### ⚠️ Never Embed Secrets in URLs
+
+URL query parameter values are redacted, but **path segments are not**. Never construct URLs like:
+
+```
+https://api.example.com/v1/token/abc123/data  # BAD — token in path
+```
+
+Instead, pass secrets via headers or request body.
+
+---
+
+## Fork Safety (Airflow / Multiprocessing)
+
+If you use `DatabaseManager` in a forked environment (e.g. Airflow workers, `multiprocessing`), you **must** either:
+
+1. Create the `DatabaseManager` **inside each worker process**, or
+2. Call `db.dispose()` **before** forking.
+
+SQLAlchemy connection pools are not safe to share across forked processes.
+
+---
+
+## Testing
+
+### Unit tests (no PostgreSQL required)
+
+```bash
+pip install -e ".[dev,dataframe]"
+pytest -v --tb=short -m "not postgres"
+```
+
+### PostgreSQL integration tests
+
+```bash
+# Start a local PostgreSQL
+docker run -d --name pg-test -e POSTGRES_PASSWORD=test -p 5432:5432 postgres:16
+
+# Run integration tests
+DATABASE_URL="postgresql+psycopg://postgres:test@localhost:5432/postgres" \
+    pytest -m postgres -v
+```
+
+### Coverage
+
+```bash
+pytest --cov=dehelpers --cov-report=term-missing -m "not postgres"
+```
+
+---
+
+## License
+
+MIT — see [LICENSE](LICENSE).

dehelpers-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+dehelpers/__init__.py,sha256=KdUCfyOQhGUhavL8nwae2UpbJ-FoT0EY3SQ9whjIPlk,645
+dehelpers/_redact.py,sha256=e5z1DcdpfRXZdXu-3jqPIjM1ZnKN2Y5e5hcnF1hZkc4,3672
+dehelpers/api.py,sha256=1HROkWMGzepuvjii2tnE99ZTDWqW3YwzbtuTv-X_T8A,13079
+dehelpers/db.py,sha256=uKY4C-X4jFc9N0osTViKNeZjXm7-Ib0vHmAdPFOLu_Q,7576
+dehelpers/exceptions.py,sha256=0dfNcLYPAMcNZI0jldGb0bF_PPXygCmB_xuKLV9QObc,1818
+dehelpers/logger.py,sha256=lfFNzAMGo7QsLS7tHvhYAC8FNkf3abxlJm7LRvQOnDU,6794
+dehelpers/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+dehelpers-0.1.0.dist-info/METADATA,sha256=gmdyU0NbXeR2Z_uR0BJV0cFSWeuw1gBWw5IFN-_1LsU,9801
+dehelpers-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+dehelpers-0.1.0.dist-info/licenses/LICENSE,sha256=eFS_S_z8RFffwvKMXclkzQlqbaMdyknmQaB549DZCBw,1072
+dehelpers-0.1.0.dist-info/RECORD,,

dehelpers-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

dehelpers-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shardul Chogale
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.