dataforge-07 0.1.0__py3-none-any.whl

dataforge/http/problem.py

@@ -0,0 +1,103 @@
+"""RFC 9457 problem details helpers for FastAPI surfaces."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from fastapi import HTTPException, Request
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ProblemDetail(BaseModel):
+    """RFC 9457 problem detail response with extension members."""
+
+    type: str = Field(default="about:blank")
+    title: str
+    status: int
+    detail: str
+    instance: str | None = None
+
+    model_config = ConfigDict(strict=True, extra="allow")
+
+
+def problem_body(
+    *,
+    status: int,
+    title: str,
+    detail: str,
+    type_: str = "about:blank",
+    instance: str | None = None,
+    **extensions: Any,
+) -> dict[str, Any]:
+    """Build a problem details JSON object."""
+    body = ProblemDetail(
+        type=type_,
+        title=title,
+        status=status,
+        detail=detail,
+        instance=instance,
+        **extensions,
+    )
+    return body.model_dump(mode="json", exclude_none=True)
+
+
+def problem_response(
+    *,
+    status: int,
+    title: str,
+    detail: str,
+    type_: str = "about:blank",
+    instance: str | None = None,
+    headers: Mapping[str, str] | None = None,
+    **extensions: Any,
+) -> JSONResponse:
+    """Return an RFC 9457 JSON response."""
+    return JSONResponse(
+        status_code=status,
+        content=problem_body(
+            status=status,
+            title=title,
+            detail=detail,
+            type_=type_,
+            instance=instance,
+            **extensions,
+        ),
+        headers=headers,
+        media_type="application/problem+json",
+    )
+
+
+async def http_exception_handler(request: Request, exc: HTTPException) -> JSONResponse:
+    """Normalize FastAPI HTTPException values into problem details."""
+    raw_detail = exc.detail
+    extensions: dict[str, Any] = {}
+    if isinstance(raw_detail, dict):
+        error_code = str(raw_detail.get("error", "http_error"))
+        message = str(raw_detail.get("message") or raw_detail.get("detail") or error_code)
+        extensions.update(raw_detail)
+    else:
+        error_code = "http_error"
+        message = str(raw_detail)
+
+    request_id = getattr(request.state, "dataforge_request_id", None)
+    if isinstance(request_id, str) and request_id and "request_id" not in extensions:
+        extensions["request_id"] = request_id
+
+    return problem_response(
+        status=exc.status_code,
+        type_=f"https://dataforge.local/problems/{error_code}",
+        title=error_code.replace("_", " ").title(),
+        detail=message,
+        instance=str(request.url.path),
+        headers=exc.headers,
+        **extensions,
+    )
+
+
+async def problem_exception_handler(request: Request, exc: Exception) -> JSONResponse:
+    """Adapter with the broad exception signature Starlette expects."""
+    if isinstance(exc, HTTPException):
+        return await http_exception_handler(request, exc)
+    raise exc

dataforge/integrations/__init__.py

@@ -0,0 +1 @@
+"""Integration package scaffolding for DataForge."""

dataforge/integrations/dbt.py

@@ -0,0 +1,164 @@
+"""dbt artifact helpers for DataForge's local repair contract.
+
+The external ``dataforge_07_dbt`` distribution can call these helpers without
+duplicating dbt manifest parsing rules.  Only dbt generic tests that DataForge
+can represent as local constraints are mapped; everything else is ignored
+conservatively.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from dataforge.verifier.schema import AcceptedValues, RelationshipConstraint, Schema
+
+
+def schema_from_dbt_artifacts(
+    manifest_path: Path,
+    *,
+    model_name: str | None = None,
+    model_unique_id: str | None = None,
+) -> Schema:
+    """Load a dbt manifest and map supported generic tests into a ``Schema``."""
+    payload = json.loads(manifest_path.read_text(encoding="utf-8"))
+    if not isinstance(payload, dict):
+        raise ValueError("dbt manifest must be a JSON object.")
+    return schema_from_dbt_manifest(
+        payload,
+        model_name=model_name,
+        model_unique_id=model_unique_id,
+    )
+
+
+def schema_from_dbt_manifest(
+    manifest: Mapping[str, Any],
+    *,
+    model_name: str | None = None,
+    model_unique_id: str | None = None,
+) -> Schema:
+    """Map supported dbt generic tests from a manifest into DataForge constraints.
+
+    Supported dbt tests:
+    - ``not_null`` -> ``Schema.not_null_columns``
+    - ``unique`` -> ``Schema.unique_columns``
+    - ``accepted_values`` -> ``Schema.accepted_values``
+    - ``relationships`` -> ``Schema.relationships``
+    """
+    target_model = _resolve_target_model(manifest, model_name, model_unique_id)
+    nodes = _mapping(manifest.get("nodes"))
+    columns = _model_columns(target_model)
+    not_null_columns: set[str] = set()
+    unique_columns: set[str] = set()
+    accepted_values: list[AcceptedValues] = []
+    relationships: list[RelationshipConstraint] = []
+
+    for raw_node in nodes.values():
+        node = _mapping(raw_node)
+        if node.get("resource_type") != "test":
+            continue
+        if target_model is not None and not _test_depends_on_model(node, target_model):
+            continue
+        metadata = _mapping(node.get("test_metadata"))
+        test_name = str(metadata.get("name", node.get("name", ""))).strip()
+        kwargs = _mapping(metadata.get("kwargs"))
+        column = _column_name(kwargs, node)
+        if not column:
+            continue
+
+        if test_name == "not_null":
+            not_null_columns.add(column)
+        elif test_name == "unique":
+            unique_columns.add(column)
+        elif test_name == "accepted_values":
+            values = _accepted_values(kwargs)
+            if values:
+                accepted_values.append(AcceptedValues(column=column, values=values))
+        elif test_name == "relationships":
+            reference = str(kwargs.get("to", "")).strip()
+            reference_column = str(kwargs.get("field", "")).strip()
+            if reference and reference_column:
+                relationships.append(
+                    RelationshipConstraint(
+                        column=column,
+                        reference=reference,
+                        reference_column=reference_column,
+                    )
+                )
+
+    return Schema(
+        columns=columns,
+        not_null_columns=frozenset(not_null_columns),
+        unique_columns=frozenset(unique_columns),
+        accepted_values=tuple(accepted_values),
+        relationships=tuple(relationships),
+    )
+
+
+def _mapping(value: object) -> Mapping[str, Any]:
+    """Return a mapping view for JSON objects, or an empty mapping."""
+    return value if isinstance(value, Mapping) else {}
+
+
+def _resolve_target_model(
+    manifest: Mapping[str, Any],
+    model_name: str | None,
+    model_unique_id: str | None,
+) -> Mapping[str, Any] | None:
+    """Resolve the optional target model node from a dbt manifest."""
+    if model_name is None and model_unique_id is None:
+        return None
+    nodes = _mapping(manifest.get("nodes"))
+    for unique_id, raw_node in nodes.items():
+        node = _mapping(raw_node)
+        if node.get("resource_type") != "model":
+            continue
+        if model_unique_id is not None and str(unique_id) == model_unique_id:
+            return {**node, "unique_id": str(unique_id)}
+        if model_name is not None and str(node.get("name", "")) == model_name:
+            return {**node, "unique_id": str(unique_id)}
+    raise ValueError("Requested dbt model was not found in manifest.")
+
+
+def _model_columns(model_node: Mapping[str, Any] | None) -> dict[str, str]:
+    """Extract declared column types from a dbt model node when available."""
+    if model_node is None:
+        return {}
+    columns: dict[str, str] = {}
+    for column_name, raw_column in _mapping(model_node.get("columns")).items():
+        column = _mapping(raw_column)
+        raw_type = column.get("data_type") or column.get("type")
+        if raw_type:
+            columns[str(column_name)] = str(raw_type)
+    return columns
+
+
+def _test_depends_on_model(
+    test_node: Mapping[str, Any],
+    model_node: Mapping[str, Any],
+) -> bool:
+    """Return whether a dbt test node depends on the target model."""
+    target_unique_id = str(model_node.get("unique_id", ""))
+    if not target_unique_id:
+        target_unique_id = str(model_node.get("name", ""))
+    depends_on = _mapping(test_node.get("depends_on"))
+    raw_nodes = depends_on.get("nodes", [])
+    if not isinstance(raw_nodes, list):
+        return False
+    return target_unique_id in {str(node) for node in raw_nodes}
+
+
+def _column_name(kwargs: Mapping[str, Any], test_node: Mapping[str, Any]) -> str:
+    """Extract the tested column name from dbt metadata variants."""
+    raw_column = kwargs.get("column_name") or kwargs.get("field") or test_node.get("column_name")
+    return str(raw_column or "").strip()
+
+
+def _accepted_values(kwargs: Mapping[str, Any]) -> tuple[str, ...]:
+    """Extract dbt accepted_values values as a stable tuple of strings."""
+    raw_values = kwargs.get("values", [])
+    if not isinstance(raw_values, list):
+        return ()
+    return tuple(str(value) for value in raw_values)

dataforge/observability.py

@@ -0,0 +1,76 @@
+"""Optional OpenTelemetry hooks for DataForge backend surfaces."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterator
+from contextlib import contextmanager, nullcontext
+from importlib import import_module
+from typing import Any
+
+_SENSITIVE_ATTR_FRAGMENTS = ("authorization", "cookie", "token", "key", "secret", "password")
+
+
+def _otel_enabled() -> bool:
+    """Return whether optional OpenTelemetry instrumentation is enabled."""
+    return os.environ.get("DATAFORGE_OTEL_ENABLED", "").strip().lower() in {
+        "1",
+        "true",
+        "yes",
+        "on",
+    }
+
+
+def _safe_attrs(attributes: dict[str, Any]) -> dict[str, str | int | float | bool]:
+    """Keep only scalar, non-sensitive telemetry attributes."""
+    safe: dict[str, str | int | float | bool] = {}
+    for key, value in attributes.items():
+        lowered = key.lower()
+        if any(fragment in lowered for fragment in _SENSITIVE_ATTR_FRAGMENTS):
+            continue
+        if lowered in {"row_values", "rows", "payload", "source_bytes", "csv"}:
+            continue
+        if isinstance(value, str | int | float | bool):
+            safe[key] = value
+    return safe
+
+
+def configure_fastapi_observability(app: Any, *, service_name: str) -> bool:
+    """Instrument a FastAPI app when OpenTelemetry is explicitly enabled."""
+    if not _otel_enabled():
+        return False
+    try:
+        fastapi_instrumentation = import_module("opentelemetry.instrumentation.fastapi")
+        trace_module = import_module("opentelemetry.trace")
+    except ImportError:
+        return False
+
+    app.state.dataforge_service_name = service_name
+    fastapi_instrumentation.FastAPIInstrumentor.instrument_app(
+        app,
+        tracer_provider=trace_module.get_tracer_provider(),
+        excluded_urls="/api/docs,/docs,/redoc,/openapi.json",
+    )
+    return True
+
+
+@contextmanager
+def repair_stage_span(stage: str, **attributes: Any) -> Iterator[None]:
+    """Create a repair-stage span when OpenTelemetry is available."""
+    if not _otel_enabled():
+        with nullcontext():
+            yield
+        return
+
+    try:
+        trace_module = import_module("opentelemetry.trace")
+    except ImportError:
+        with nullcontext():
+            yield
+        return
+
+    tracer = trace_module.get_tracer("dataforge.repair")
+    with tracer.start_as_current_span(stage) as span:
+        for key, value in _safe_attrs(attributes).items():
+            span.set_attribute(key, value)
+        yield

dataforge/py.typed

@@ -0,0 +1 @@
+

dataforge/release/__init__.py

@@ -0,0 +1 @@
+"""Release verification helpers for DataForge."""