parquetscope 0.3.0__py3-none-any.whl

parquetscope/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

parquetscope/analysis.py

@@ -0,0 +1,176 @@
+from .intervals import coverage, union
+from .models import Capture, HTTPObservation, Metrics, ServerObservation, span
+from .models_v2 import (
+    FileHTTPObservation,
+    FileInput,
+    FileMetrics,
+    FileServerObservation,
+    SetMetrics,
+)
+
+
+def derive_metrics(
+    observations: list[HTTPObservation], server: list[ServerObservation], capture: Capture
+) -> Metrics:
+    gets = [o for o in observations if o.method == "GET"]
+    requested = [o.requested_span.pair() for o in gets if o.requested_span]
+    confirmed = [o.response_span.pair() for o in gets if o.response_span]
+    known_requests = all(o.requested_span is not None for o in gets)
+    known_responses = all(o.response_span is not None for o in gets)
+    known_lengths = all(o.declared_body_bytes is not None for o in gets)
+    reasons = []
+    if not capture.complete:
+        reasons.append(
+            capture.reason or "Incomplete capture; metrics describe observed subset only"
+        )
+    if not known_requests:
+        reasons.append("Some GETs have no interpretable single requested range")
+    if not known_responses:
+        reasons.append("Some GET responses have no confirmed file interval")
+    if not known_lengths:
+        reasons.append("Some GET responses have no declared Content-Length")
+    total = sum(b - a for a, b in confirmed)
+    unique = coverage(confirmed)
+    return Metrics(
+        get_count=len(gets),
+        head_count=sum(o.method == "HEAD" for o in observations),
+        requested_range_bytes=str(sum(b - a for a, b in requested)) if known_requests else None,
+        confirmed_range_bytes=str(total) if known_responses else None,
+        declared_body_bytes=(
+            str(sum(int(o.declared_body_bytes) for o in gets if o.declared_body_bytes is not None))
+            if known_lengths
+            else None
+        ),
+        server_written_bytes=str(sum(int(o.written_body_bytes) for o in server)),
+        unique_coverage_bytes=str(unique) if known_responses else None,
+        repeated_coverage_bytes=str(total - unique) if known_responses else None,
+        coverage=[span(a, b) for a, b in union(confirmed)] if known_responses else None,
+        totals_scope="complete" if capture.complete else "observed_subset",
+        unknown_reasons=reasons,
+    )
+
+
+def reconcile(observations: list[HTTPObservation], server: list[ServerObservation]) -> bool:
+    """Response IDs distinguish repeated and concurrent identical requests."""
+    return _reconcile(observations, server, lambda _o, _s: True)
+
+
+def reconcile_set(
+    observations: list[FileHTTPObservation], server: list[FileServerObservation]
+) -> bool:
+    """As `reconcile`, and both independent sources must attribute a request to the same file."""
+    return _reconcile(observations, server, lambda o, s: o.file_id == s.file_id)
+
+
+def _reconcile(observations, server, same_file) -> bool:
+    by_id = {o.id: o for o in server}
+    if (
+        len(by_id) != len(server)
+        or len(server) != len(observations)
+        or len({o.id for o in observations}) != len(observations)
+    ):
+        return False
+    for o in observations:
+        s = by_id.get(o.id)
+        if not s or (o.method, o.requested_range, o.response_span, o.status) != (
+            s.method,
+            s.requested_range,
+            s.response_span,
+            s.status,
+        ):
+            return False
+        if o.declared_body_bytes != s.declared_body_bytes or not same_file(o, s):
+            return False
+    return True
+
+
+def _file_metrics(
+    file_id: str, observations: list[FileHTTPObservation], server: list[FileServerObservation]
+) -> FileMetrics:
+    """Intervals belong to one file; callers never pass observations of several files."""
+    gets = [o for o in observations if o.method == "GET"]
+    heads = len(observations) - len(gets)
+    requested = [o.requested_span.pair() for o in gets if o.requested_span]
+    confirmed = [o.response_span.pair() for o in gets if o.response_span]
+    known_requests = all(o.requested_span is not None for o in gets)
+    known_responses = all(o.response_span is not None for o in gets)
+    known_lengths = all(o.declared_body_bytes is not None for o in gets)
+    reasons = []
+    if not known_requests:
+        reasons.append("Some GETs have no interpretable single requested range")
+    if not known_responses:
+        reasons.append("Some GET responses have no confirmed file interval")
+    if not known_lengths:
+        reasons.append("Some GET responses have no declared Content-Length")
+    total = sum(b - a for a, b in confirmed)
+    unique = coverage(confirmed)
+    # Either independent source is enough to say a request for the file was observed.
+    server_gets = any(s.method == "GET" for s in server)
+    server_heads = any(s.method == "HEAD" for s in server)
+    if gets or server_gets:
+        access = "get"
+    elif heads or server_heads:
+        access = "head_only"
+    else:
+        access = "none"
+    return FileMetrics(
+        file_id=file_id,
+        observed_access=access,
+        get_count=len(gets),
+        head_count=heads,
+        requested_range_bytes=str(sum(b - a for a, b in requested)) if known_requests else None,
+        confirmed_range_bytes=str(total) if known_responses else None,
+        declared_body_bytes=(
+            str(sum(int(o.declared_body_bytes) for o in gets if o.declared_body_bytes is not None))
+            if known_lengths
+            else None
+        ),
+        server_written_bytes=str(sum(int(o.written_body_bytes) for o in server)),
+        unique_coverage_bytes=str(unique) if known_responses else None,
+        repeated_coverage_bytes=str(total - unique) if known_responses else None,
+        coverage=[span(a, b) for a, b in union(confirmed)] if known_responses else None,
+        unknown_reasons=reasons,
+    )
+
+
+def _sum_known(values: list[str | None]) -> str | None:
+    return None if any(v is None for v in values) else str(sum(int(v) for v in values))
+
+
+def derive_set_metrics(
+    files: list[FileInput],
+    observations: list[FileHTTPObservation],
+    server: list[FileServerObservation],
+    capture: Capture,
+) -> SetMetrics:
+    """Per-file metrics grouped by file_id in one pass; set totals are sums of file totals."""
+    by_file: dict[str, list[FileHTTPObservation]] = {f.id: [] for f in files}
+    server_by_file: dict[str, list[FileServerObservation]] = {f.id: [] for f in files}
+    for o in observations:
+        by_file[o.file_id].append(o)
+    for s in server:
+        server_by_file[s.file_id].append(s)
+    per_file = [_file_metrics(f.id, by_file[f.id], server_by_file[f.id]) for f in files]
+    reasons = []
+    if not capture.complete:
+        reasons.append(
+            capture.reason or "Incomplete capture; metrics describe observed subset only"
+        )
+    for m in per_file:
+        reasons.extend(r for r in m.unknown_reasons if r not in reasons)
+    return SetMetrics(
+        files_without_observed_access=sum(m.observed_access == "none" for m in per_file),
+        files_head_only=sum(m.observed_access == "head_only" for m in per_file),
+        files_with_get=sum(m.observed_access == "get" for m in per_file),
+        get_count=sum(m.get_count for m in per_file),
+        head_count=sum(m.head_count for m in per_file),
+        requested_range_bytes=_sum_known([m.requested_range_bytes for m in per_file]),
+        confirmed_range_bytes=_sum_known([m.confirmed_range_bytes for m in per_file]),
+        declared_body_bytes=_sum_known([m.declared_body_bytes for m in per_file]),
+        server_written_bytes=str(sum(int(m.server_written_bytes) for m in per_file)),
+        unique_coverage_bytes=_sum_known([m.unique_coverage_bytes for m in per_file]),
+        repeated_coverage_bytes=_sum_known([m.repeated_coverage_bytes for m in per_file]),
+        totals_scope="complete" if capture.complete else "observed_subset",
+        unknown_reasons=reasons,
+        files=per_file,
+    )

parquetscope/app.py

@@ -0,0 +1,204 @@
+import asyncio
+import threading
+import uuid
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Literal
+from urllib.parse import urlparse
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+from pydantic import Field
+
+from .capture import capture, capture_set
+from .demo import SQL, generate
+from .engines import SUPPORTED_ENGINES, engine_status
+from .manifest import SetSpec
+from .models import MAX_ARTIFACT_BYTES, Model, Run, Settings, load_artifact
+from .models_v2 import SetRun
+from .models_v3 import Settings3
+from .offline import STATIC, viewer_html
+
+
+class RunRequest(Model):
+    input_id: str
+    sql: str = Field(min_length=1, max_length=65536)
+    settings: Settings3 = Field(
+        default_factory=lambda: Settings3(
+            threads=4,
+            memory_limit_mb=512,
+            deadline_seconds=30,
+            preview_rows=30,
+        )
+    )
+    # None: the project's own interpreter. A version needs `parquetscope setup --engine` first.
+    engine: Literal[SUPPORTED_ENGINES] | None = None
+
+
+def describe_inputs(inputs: dict[str, "Path | SetSpec"]) -> list[dict]:
+    return [
+        {"id": i, "name": p.label, "kind": "set", "file_count": len(p.entries)}
+        if isinstance(p, SetSpec)
+        else {"id": i, "name": p.name, "kind": "file", "file_count": 1}
+        for i, p in inputs.items()
+    ]
+
+
+def create_app(
+    paths: list[Path] | None = None,
+    artifacts: list[Run | SetRun] | None = None,
+    demo_directory: Path | None = None,
+    manifests: list[SetSpec] | None = None,
+) -> FastAPI:
+    inputs: dict[str, Path | SetSpec] = {str(i): p.resolve() for i, p in enumerate(paths or [])}
+    inputs.update({f"set-{i}": spec for i, spec in enumerate(manifests or [])})
+    jobs: dict[str, dict] = {}
+    stored = {r.id: r for r in artifacts or []}
+    tasks: set[asyncio.Task] = set()
+
+    @asynccontextmanager
+    async def lifespan(_app):
+        yield
+        for job in jobs.values():
+            job["cancel"].set()
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    app = FastAPI(title="ParquetScope", lifespan=lifespan, docs_url=None, redoc_url=None)
+
+    @app.middleware("http")
+    async def loopback_boundary(request: Request, call_next):
+        host = request.url.hostname
+        if host not in ("127.0.0.1", "localhost", "::1", "testserver"):
+            return JSONResponse({"detail": "Loopback hosts only"}, status_code=403)
+        origin = request.headers.get("origin")
+        if origin and urlparse(origin).netloc != request.headers.get("host"):
+            return JSONResponse(
+                {"detail": "Cross-origin requests are not allowed"}, status_code=403
+            )
+        response = await call_next(request)
+        response.headers["X-Content-Type-Options"] = "nosniff"
+        response.headers["Referrer-Policy"] = "no-referrer"
+        return response
+
+    @app.get("/api/session")
+    def session():
+        return {
+            "inputs": describe_inputs(inputs),
+            "runs": [r.model_dump() for r in stored.values()],
+            "demo_available": demo_directory is not None,
+            "engines": engine_status(),
+        }
+
+    @app.post("/api/demo")
+    async def demo():
+        if demo_directory is None:
+            raise HTTPException(403, "Start with parquetscope serve --demo to enable the demo")
+        if any(j["status"] == "running" for j in jobs.values()):
+            raise HTTPException(409, "Wait for the current run to finish")
+        original, sorted_path, _ = await asyncio.to_thread(generate, demo_directory)
+        inputs["demo-original"], inputs["demo-sorted"] = original, sorted_path
+        return {"inputs": describe_inputs(inputs), "sql": SQL}
+
+    async def work(job_id: str, payload: RunRequest):
+        job = jobs[job_id]
+        try:
+            source = inputs[payload.input_id]
+            if isinstance(source, SetSpec):
+                run, preview = await asyncio.to_thread(
+                    capture_set,
+                    source,
+                    payload.sql,
+                    payload.settings,
+                    job["cancel"],
+                    payload.engine,
+                )
+            else:
+                run, preview = await asyncio.to_thread(
+                    capture,
+                    source,
+                    payload.sql,
+                    Settings(
+                        **payload.settings.model_dump(exclude={"disable_parquet_prefetching"})
+                    ),
+                    job["cancel"],
+                )
+            stored[run.id] = run
+            job.update(status="finished", run=run.model_dump(), preview=preview)
+        except Exception as error:
+            job.update(status="error", error=str(error))
+
+    @app.post("/api/runs", status_code=202)
+    async def start(payload: RunRequest):
+        if payload.input_id not in inputs:
+            raise HTTPException(404, "Input is not explicitly selected for this session")
+        if not isinstance(inputs[payload.input_id], SetSpec) and (
+            payload.engine or payload.settings.disable_parquet_prefetching is not None
+        ):
+            raise HTTPException(422, "Engine and prefetch selection need a file-set input")
+        if any(j["status"] == "running" for j in jobs.values()):
+            raise HTTPException(409, "One capture at a time")
+        if len(stored) >= 20:
+            raise HTTPException(409, "Session limit: 20 runs. Export artifacts and restart.")
+        job_id = uuid.uuid4().hex
+        jobs[job_id] = {"status": "running", "cancel": threading.Event()}
+        task = asyncio.create_task(work(job_id, payload))
+        tasks.add(task)
+        task.add_done_callback(tasks.discard)
+        return {"id": job_id}
+
+    @app.get("/api/jobs/{job_id}")
+    def poll(job_id: str):
+        if job_id not in jobs:
+            raise HTTPException(404, "Unknown run")
+        return {key: value for key, value in jobs[job_id].items() if key != "cancel"}
+
+    @app.post("/api/jobs/{job_id}/cancel")
+    def cancel(job_id: str):
+        if job_id not in jobs:
+            raise HTTPException(404, "Unknown run")
+        jobs[job_id]["cancel"].set()
+        return {"status": "cancelling"}
+
+    @app.post("/api/import")
+    async def import_run(request: Request):
+        chunks = bytearray()
+        async for chunk in request.stream():
+            chunks.extend(chunk)
+            if len(chunks) > MAX_ARTIFACT_BYTES:
+                raise HTTPException(413, "Artifact exceeds 32 MiB limit")
+        try:
+            run = load_artifact(bytes(chunks))
+        except ValueError as error:
+            raise HTTPException(422, str(error)) from error
+        if len(stored) >= 20:
+            raise HTTPException(409, "Session limit: 20 runs")
+        stored[run.id] = run
+        return run.model_dump()
+
+    @app.get("/api/artifact/{run_id}")
+    def artifact(run_id: str):
+        if run_id not in stored:
+            raise HTTPException(404, "Unknown run")
+        return JSONResponse(
+            stored[run_id].model_dump(),
+            headers={
+                "Content-Disposition": f'attachment; filename="{run_id}.parquetscope.json"',
+            },
+        )
+
+    @app.get("/offline.html")
+    def offline():
+        return HTMLResponse(viewer_html())
+
+    @app.get("/")
+    def index():
+        if not (STATIC / "index.html").exists():
+            raise HTTPException(503, "Frontend missing. Run pnpm --dir frontend build")
+        return FileResponse(STATIC / "index.html")
+
+    if (STATIC / "assets").exists():
+        app.mount("/assets", StaticFiles(directory=STATIC / "assets"))
+    app.state.jobs = jobs
+    return app