PyPI - dataforge-07 - Versions diffs - 0.1.0__py3-none-any.whl - Mend

dataforge-07 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (150) hide show

dataforge/__init__.py +204 -0
dataforge/__main__.py +5 -0
dataforge/agent/__init__.py +16 -0
dataforge/agent/providers.py +259 -0
dataforge/agent/scratchpad.py +183 -0
dataforge/agent/tool_actions.py +343 -0
dataforge/bench/__init__.py +31 -0
dataforge/bench/core.py +426 -0
dataforge/bench/groq_client.py +386 -0
dataforge/bench/methods.py +443 -0
dataforge/bench/report.py +309 -0
dataforge/bench/runner.py +247 -0
dataforge/causal/__init__.py +21 -0
dataforge/causal/dag.py +174 -0
dataforge/causal/pc.py +232 -0
dataforge/causal/root_cause.py +193 -0
dataforge/cli/__init__.py +50 -0
dataforge/cli/audit.py +70 -0
dataforge/cli/bench.py +154 -0
dataforge/cli/common.py +267 -0
dataforge/cli/constraints.py +407 -0
dataforge/cli/profile.py +147 -0
dataforge/cli/release.py +166 -0
dataforge/cli/repair.py +407 -0
dataforge/cli/revert.py +139 -0
dataforge/cli/watch.py +144 -0
dataforge/datasets/__init__.py +25 -0
dataforge/datasets/embedded/hospital/clean.csv +11 -0
dataforge/datasets/embedded/hospital/dirty.csv +11 -0
dataforge/datasets/real_world.py +290 -0
dataforge/datasets/registry.py +103 -0
dataforge/detectors/__init__.py +80 -0
dataforge/detectors/base.py +145 -0
dataforge/detectors/decimal_shift.py +166 -0
dataforge/detectors/fd_violation.py +157 -0
dataforge/detectors/type_mismatch.py +173 -0
dataforge/engine/__init__.py +39 -0
dataforge/engine/repair.py +905 -0
dataforge/env/__init__.py +22 -0
dataforge/env/environment.py +883 -0
dataforge/env/observation.py +61 -0
dataforge/env/openenv_core.py +161 -0
dataforge/env/reward.py +128 -0
dataforge/env/server.py +176 -0
dataforge/evaluation_contract.py +76 -0
dataforge/fixtures/hospital_10rows.csv +11 -0
dataforge/fixtures/hospital_schema.yaml +17 -0
dataforge/http/__init__.py +1 -0
dataforge/http/problem.py +103 -0
dataforge/integrations/__init__.py +1 -0
dataforge/integrations/dbt.py +164 -0
dataforge/observability.py +76 -0
dataforge/py.typed +1 -0
dataforge/release/__init__.py +1 -0
dataforge/release/doctor.py +367 -0
dataforge/release/full_vision.py +702 -0
dataforge/release/gate.py +861 -0
dataforge/release/playground_check.py +411 -0
dataforge/repair_contract.py +468 -0
dataforge/repairers/__init__.py +88 -0
dataforge/repairers/base.py +77 -0
dataforge/repairers/decimal_shift.py +43 -0
dataforge/repairers/fd_violation.py +225 -0
dataforge/repairers/type_mismatch.py +73 -0
dataforge/safety/__init__.py +5 -0
dataforge/safety/adversarial/attack_01_phone_pii.yaml +8 -0
dataforge/safety/adversarial/attack_02_phone_pii.yaml +8 -0
dataforge/safety/adversarial/attack_03_phone_pii.yaml +8 -0
dataforge/safety/adversarial/attack_04_phone_pii.yaml +8 -0
dataforge/safety/adversarial/attack_05_phone_pii.yaml +8 -0
dataforge/safety/adversarial/attack_06_phone_pii.yaml +8 -0
dataforge/safety/adversarial/attack_07_phone_pii.yaml +8 -0
dataforge/safety/adversarial/attack_08_phone_pii.yaml +8 -0
dataforge/safety/adversarial/attack_09_phone_pii.yaml +8 -0
dataforge/safety/adversarial/attack_10_phone_pii.yaml +8 -0
dataforge/safety/adversarial/attack_11_ssn_pii.yaml +8 -0
dataforge/safety/adversarial/attack_12_ssn_pii.yaml +8 -0
dataforge/safety/adversarial/attack_13_ssn_pii.yaml +8 -0
dataforge/safety/adversarial/attack_14_ssn_pii.yaml +8 -0
dataforge/safety/adversarial/attack_15_ssn_pii.yaml +8 -0
dataforge/safety/adversarial/attack_16_ssn_pii.yaml +8 -0
dataforge/safety/adversarial/attack_17_ssn_pii.yaml +8 -0
dataforge/safety/adversarial/attack_18_ssn_pii.yaml +8 -0
dataforge/safety/adversarial/attack_19_ssn_pii.yaml +8 -0
dataforge/safety/adversarial/attack_20_ssn_pii.yaml +8 -0
dataforge/safety/adversarial/attack_21_email_pii.yaml +8 -0
dataforge/safety/adversarial/attack_22_email_pii.yaml +8 -0
dataforge/safety/adversarial/attack_23_email_pii.yaml +8 -0
dataforge/safety/adversarial/attack_24_email_pii.yaml +8 -0
dataforge/safety/adversarial/attack_25_email_pii.yaml +8 -0
dataforge/safety/adversarial/attack_26_email_pii.yaml +8 -0
dataforge/safety/adversarial/attack_27_email_pii.yaml +8 -0
dataforge/safety/adversarial/attack_28_email_pii.yaml +8 -0
dataforge/safety/adversarial/attack_29_email_pii.yaml +8 -0
dataforge/safety/adversarial/attack_30_email_pii.yaml +8 -0
dataforge/safety/adversarial/attack_31_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_32_row_delete.yaml +8 -0
dataforge/safety/adversarial/attack_33_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_34_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_35_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_36_row_delete.yaml +11 -0
dataforge/safety/adversarial/attack_37_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_38_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_39_row_delete.yaml +8 -0
dataforge/safety/adversarial/attack_40_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_41_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_42_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_43_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_44_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_45_row_delete.yaml +8 -0
dataforge/safety/adversarial/attack_46_row_delete.yaml +8 -0
dataforge/safety/adversarial/attack_47_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_48_row_delete.yaml +7 -0
dataforge/safety/adversarial/attack_49_row_delete.yaml +8 -0
dataforge/safety/adversarial/attack_50_row_delete.yaml +7 -0
dataforge/safety/constitution.py +307 -0
dataforge/safety/constitutions/default.yaml +40 -0
dataforge/safety/filter.py +134 -0
dataforge/schema_inference.py +620 -0
dataforge/stores/__init__.py +46 -0
dataforge/stores/base.py +73 -0
dataforge/stores/cloud.py +78 -0
dataforge/stores/csv.py +94 -0
dataforge/stores/duckdb.py +313 -0
dataforge/stores/patch_plan.py +178 -0
dataforge/stores/registry.py +82 -0
dataforge/stores/repair.py +121 -0
dataforge/stores/revert.py +22 -0
dataforge/stores/sql.py +27 -0
dataforge/table.py +228 -0
dataforge/transactions/__init__.py +34 -0
dataforge/transactions/files.py +96 -0
dataforge/transactions/log.py +613 -0
dataforge/transactions/revert.py +102 -0
dataforge/transactions/txn.py +104 -0
dataforge/ui/__init__.py +1 -0
dataforge/ui/profile_view.py +136 -0
dataforge/ui/repair_diff.py +91 -0
dataforge/verifier/__init__.py +55 -0
dataforge/verifier/constraint_ir.py +155 -0
dataforge/verifier/explain.py +47 -0
dataforge/verifier/gate.py +5 -0
dataforge/verifier/schema.py +111 -0
dataforge/verifier/smt.py +433 -0
dataforge_07-0.1.0.dist-info/METADATA +436 -0
dataforge_07-0.1.0.dist-info/RECORD +150 -0
dataforge_07-0.1.0.dist-info/WHEEL +5 -0
dataforge_07-0.1.0.dist-info/entry_points.txt +3 -0
dataforge_07-0.1.0.dist-info/licenses/LICENSE +176 -0
dataforge_07-0.1.0.dist-info/top_level.txt +1 -0

dataforge/release/playground_check.py ADDED Viewed

@@ -0,0 +1,411 @@
+"""Live Playground release and monitoring checks."""
+from __future__ import annotations
+import json
+import time
+from dataclasses import asdict, dataclass
+from typing import Any
+from urllib.parse import urlsplit
+import httpx
+from dataforge.release.doctor import run_doctor
+DEFAULT_BACKEND_URL = "https://Praneshrajan15-dataforge-playground.hf.space"
+DEFAULT_FRONTEND_URL = "https://dataforge.praneshrajan15.workers.dev/playground"
+NEGATIVE_CORS_ORIGIN = "https://untrusted-dataforge.example"
+REQUIRED_HEALTH_KEYS = {"status", "advanced_available", "max_upload_bytes"}
+ENHANCED_HEALTH_KEYS = {
+    "service",
+    "api_version",
+    "contract_version",
+    "build_sha",
+    "server_time_utc",
+    "environment",
+    "limits",
+    "cors_configured",
+    "otel_enabled",
+    "metrics",
+}
+@dataclass(frozen=True)
+class PlaygroundCheck:
+    """One Playground check result."""
+    name: str
+    ok: bool
+    detail: str
+    metadata: dict[str, Any]
+@dataclass(frozen=True)
+class PlaygroundCheckReport:
+    """Machine-readable Playground release report."""
+    ok: bool
+    frontend_url: str
+    backend_url: str
+    checks: list[PlaygroundCheck]
+    def to_dict(self) -> dict[str, Any]:
+        """Return a JSON-serializable report."""
+        return asdict(self)
+def normalize_url(value: str) -> str:
+    """Strip whitespace and trailing slashes from a URL."""
+    return value.strip().rstrip("/")
+def frontend_origin(frontend_url: str) -> str:
+    """Return the origin portion of a frontend URL."""
+    parts = urlsplit(frontend_url)
+    return f"{parts.scheme}://{parts.netloc}"
+def join_url(base_url: str, path: str) -> str:
+    """Join a normalized base URL and absolute path fragment."""
+    return f"{base_url.rstrip('/')}/{path.lstrip('/')}"
+def _timed_request(
+    client: httpx.Client, method: str, url: str, **kwargs: Any
+) -> tuple[httpx.Response, float]:
+    """Run one HTTP request and return response plus elapsed milliseconds."""
+    started = time.perf_counter()
+    response = client.request(method, url, **kwargs)
+    return response, (time.perf_counter() - started) * 1000
+def _check_frontend_deployed(
+    client: httpx.Client,
+    *,
+    frontend_url: str,
+) -> PlaygroundCheck:
+    try:
+        response, latency_ms = _timed_request(client, "GET", frontend_url)
+        body = response.text
+        ok = (
+            response.status_code == 200
+            and "<!doctype html>" in body.lower()
+            and 'id="root"' in body
+            and "config.js" in body
+            and "/playground/assets/" in body
+        )
+        detail = (
+            "Frontend serves the React shell."
+            if ok
+            else "Frontend shell is missing required markers."
+        )
+        return PlaygroundCheck(
+            "frontend_deployed",
+            ok,
+            detail,
+            {
+                "status_code": response.status_code,
+                "latency_ms": round(latency_ms, 2),
+                "content_type": response.headers.get("content-type", ""),
+            },
+        )
+    except Exception as exc:
+        return PlaygroundCheck("frontend_deployed", False, str(exc), {})
+def _check_config_js(
+    client: httpx.Client,
+    *,
+    frontend_url: str,
+    backend_url: str,
+) -> PlaygroundCheck:
+    try:
+        response, latency_ms = _timed_request(client, "GET", join_url(frontend_url, "config.js"))
+        cache_control = response.headers.get("cache-control", "")
+        ok = (
+            response.status_code == 200
+            and backend_url in response.text
+            and "no-store" in cache_control.lower()
+        )
+        detail = (
+            "config.js points at the expected backend and is uncached."
+            if ok
+            else "config.js is stale or cacheable."
+        )
+        return PlaygroundCheck(
+            "config_js_correct",
+            ok,
+            detail,
+            {
+                "status_code": response.status_code,
+                "latency_ms": round(latency_ms, 2),
+                "cache_control": cache_control,
+            },
+        )
+    except Exception as exc:
+        return PlaygroundCheck("config_js_correct", False, str(exc), {})
+def _check_backend_deployed(
+    client: httpx.Client,
+    *,
+    backend_url: str,
+    latency_threshold_ms: float,
+) -> PlaygroundCheck:
+    try:
+        root, root_latency_ms = _timed_request(client, "GET", backend_url)
+        health, health_latency_ms = _timed_request(client, "GET", f"{backend_url}/api/health")
+        payload = health.json() if health.status_code == 200 else {}
+        missing = sorted(REQUIRED_HEALTH_KEYS - set(payload))
+        enhanced_missing = sorted(ENHANCED_HEALTH_KEYS - set(payload))
+        ok = (
+            root.status_code == 200
+            and health.status_code == 200
+            and payload.get("status") == "ok"
+            and not missing
+            and health_latency_ms <= latency_threshold_ms
+        )
+        detail = (
+            "Backend root and health are reachable."
+            if ok
+            else "Backend root or health check failed."
+        )
+        return PlaygroundCheck(
+            "backend_deployed",
+            ok,
+            detail,
+            {
+                "root_status_code": root.status_code,
+                "health_status_code": health.status_code,
+                "root_latency_ms": round(root_latency_ms, 2),
+                "health_latency_ms": round(health_latency_ms, 2),
+                "latency_threshold_ms": latency_threshold_ms,
+                "required_missing": missing,
+                "enhanced_missing": enhanced_missing,
+                "metrics": payload.get("metrics", {}),
+            },
+        )
+    except Exception as exc:
+        return PlaygroundCheck("backend_deployed", False, str(exc), {})
+def _check_cors(
+    client: httpx.Client,
+    *,
+    frontend_url: str,
+    backend_url: str,
+) -> PlaygroundCheck:
+    origin = frontend_origin(frontend_url)
+    try:
+        positive, positive_latency_ms = _timed_request(
+            client,
+            "GET",
+            f"{backend_url}/api/health",
+            headers={"Origin": origin},
+        )
+        negative, negative_latency_ms = _timed_request(
+            client,
+            "GET",
+            f"{backend_url}/api/health",
+            headers={"Origin": NEGATIVE_CORS_ORIGIN},
+        )
+        preflight, preflight_latency_ms = _timed_request(
+            client,
+            "OPTIONS",
+            f"{backend_url}/api/health",
+            headers={
+                "Origin": NEGATIVE_CORS_ORIGIN,
+                "Access-Control-Request-Method": "GET",
+            },
+        )
+        allowed_origin = positive.headers.get("access-control-allow-origin")
+        negative_allowed_origin = negative.headers.get("access-control-allow-origin")
+        preflight_allowed_origin = preflight.headers.get("access-control-allow-origin")
+        negative_error = ""
+        try:
+            negative_error = str(negative.json().get("error", ""))
+        except ValueError:
+            negative_error = ""
+        negative_denied = negative.status_code == 403 and negative_error == "origin_not_allowed"
+        ok = allowed_origin == origin and positive.status_code == 200 and negative_denied
+        detail = (
+            "Configured origin is allowed and disallowed origins cannot read API data."
+            if ok
+            else "CORS origin enforcement is incorrect."
+        )
+        return PlaygroundCheck(
+            "cors_correct",
+            ok,
+            detail,
+            {
+                "frontend_origin": origin,
+                "positive_status_code": positive.status_code,
+                "negative_status_code": negative.status_code,
+                "allowed_origin": allowed_origin,
+                "negative_allowed_origin": negative_allowed_origin,
+                "negative_error": negative_error,
+                "negative_preflight_status_code": preflight.status_code,
+                "negative_preflight_allowed_origin": preflight_allowed_origin,
+                "positive_latency_ms": round(positive_latency_ms, 2),
+                "negative_latency_ms": round(negative_latency_ms, 2),
+                "negative_preflight_latency_ms": round(preflight_latency_ms, 2),
+            },
+        )
+    except Exception as exc:
+        return PlaygroundCheck("cors_correct", False, str(exc), {})
+def _check_doctor() -> PlaygroundCheck:
+    report = run_doctor(core=True, maintainer_deploy=False)
+    return PlaygroundCheck(
+        "doctor_passing",
+        report.ok,
+        "Core release doctor passed." if report.ok else "Core release doctor failed.",
+        {"doctor": report.to_dict()},
+    )
+def _check_smoke_flow(client: httpx.Client, *, backend_url: str) -> PlaygroundCheck:
+    try:
+        sample, sample_latency_ms = _timed_request(
+            client,
+            "GET",
+            f"{backend_url}/api/samples/hospital_10rows",
+        )
+        if sample.status_code != 200:
+            return PlaygroundCheck(
+                "smoke_flow_passing",
+                False,
+                "Sample endpoint failed.",
+                {"sample_status_code": sample.status_code},
+            )
+        files = {"file": ("hospital_10rows.csv", sample.content, "text/csv")}
+        analyze, analyze_latency_ms = _timed_request(
+            client,
+            "POST",
+            f"{backend_url}/api/analyze",
+            files=files,
+        )
+        files = {"file": ("hospital_10rows.csv", sample.content, "text/csv")}
+        profile, profile_latency_ms = _timed_request(
+            client,
+            "POST",
+            f"{backend_url}/api/profile",
+            files=files,
+        )
+        files = {"file": ("hospital_10rows.csv", sample.content, "text/csv")}
+        repair, repair_latency_ms = _timed_request(
+            client,
+            "POST",
+            f"{backend_url}/api/repair",
+            files=files,
+        )
+        analyze_payload = analyze.json() if analyze.status_code == 200 else {}
+        profile_payload = profile.json() if profile.status_code == 200 else {}
+        repair_payload = repair.json() if repair.status_code == 200 else {}
+        analyze_required_keys = {
+            "source",
+            "risk_summary",
+            "repairs",
+            "verification",
+            "receipt",
+            "apply_handoff",
+        }
+        analyze_missing = sorted(analyze_required_keys - set(analyze_payload))
+        ok = (
+            analyze.status_code == 200
+            and profile.status_code == 200
+            and repair.status_code == 200
+            and not analyze_missing
+            and "issues" in profile_payload
+            and "fixes" in repair_payload
+            and "txn_journal" in repair_payload
+        )
+        return PlaygroundCheck(
+            "smoke_flow_passing",
+            ok,
+            (
+                "Sample analyze, profile, and repair dry-run passed."
+                if ok
+                else "Sample smoke flow failed."
+            ),
+            {
+                "sample_latency_ms": round(sample_latency_ms, 2),
+                "analyze_status_code": analyze.status_code,
+                "profile_status_code": profile.status_code,
+                "repair_status_code": repair.status_code,
+                "analyze_latency_ms": round(analyze_latency_ms, 2),
+                "profile_latency_ms": round(profile_latency_ms, 2),
+                "repair_latency_ms": round(repair_latency_ms, 2),
+                "analyze_missing": analyze_missing,
+                "issue_count": len(profile_payload.get("issues", []))
+                if isinstance(profile_payload, dict)
+                else None,
+                "repair_count": len(analyze_payload.get("repairs", []))
+                if isinstance(analyze_payload, dict)
+                else None,
+                "fix_count": len(repair_payload.get("fixes", []))
+                if isinstance(repair_payload, dict)
+                else None,
+            },
+        )
+    except Exception as exc:
+        return PlaygroundCheck("smoke_flow_passing", False, str(exc), {})
+def run_playground_check(
+    *,
+    frontend_url: str = DEFAULT_FRONTEND_URL,
+    backend_url: str = DEFAULT_BACKEND_URL,
+    latency_threshold_ms: float = 5_000.0,
+    include_doctor: bool = True,
+    include_smoke: bool = True,
+    client: httpx.Client | None = None,
+) -> PlaygroundCheckReport:
+    """Run the public Playground release checklist."""
+    normalized_frontend_url = normalize_url(frontend_url)
+    normalized_backend_url = normalize_url(backend_url)
+    def collect(active_client: httpx.Client) -> list[PlaygroundCheck]:
+        checks = [
+            _check_frontend_deployed(active_client, frontend_url=normalized_frontend_url),
+            _check_config_js(
+                active_client,
+                frontend_url=normalized_frontend_url,
+                backend_url=normalized_backend_url,
+            ),
+            _check_backend_deployed(
+                active_client,
+                backend_url=normalized_backend_url,
+                latency_threshold_ms=latency_threshold_ms,
+            ),
+            _check_cors(
+                active_client,
+                frontend_url=normalized_frontend_url,
+                backend_url=normalized_backend_url,
+            ),
+        ]
+        if include_doctor:
+            checks.append(_check_doctor())
+        if include_smoke:
+            checks.append(_check_smoke_flow(active_client, backend_url=normalized_backend_url))
+        return checks
+    if client is not None:
+        checks = collect(client)
+    else:
+        with httpx.Client(follow_redirects=True, timeout=30.0) as owned_client:
+            checks = collect(owned_client)
+    return PlaygroundCheckReport(
+        ok=all(check.ok for check in checks),
+        frontend_url=normalized_frontend_url,
+        backend_url=normalized_backend_url,
+        checks=checks,
+    )
+def report_to_json(report: PlaygroundCheckReport) -> str:
+    """Render a stable JSON report."""
+    return json.dumps(report.to_dict(), indent=2, sort_keys=True)