prepro-auto 1.0.0__py3-none-any.whl

app/api/router.py

@@ -0,0 +1,18 @@
+"""
+Aggregates all route modules into a single APIRouter that main.py mounts
+under the API_PREFIX (e.g. /api/v1).
+"""
+
+from fastapi import APIRouter
+
+from app.api.routes import health, datasets, decisions, execution, export, transform
+from app.api.routes import ml
+
+api_router = APIRouter()
+api_router.include_router(health.router)
+api_router.include_router(datasets.router)
+api_router.include_router(decisions.router)
+api_router.include_router(execution.router)
+api_router.include_router(export.router)
+api_router.include_router(transform.router)
+api_router.include_router(ml.router)

app/api/routes/datasets.py

@@ -0,0 +1,487 @@
+"""
+Dataset routes: upload a file and inspect a job.
+
+POST /api/v1/datasets/upload
+    Accepts a multipart file + optional domain. Validates extension and size,
+    stores the raw bytes, creates a PipelineJob row, returns the job_id.
+
+GET /api/v1/datasets/{job_id}
+    Returns the full job detail.
+"""
+
+import uuid
+
+from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.logging import get_logger
+from app.db.session import get_db
+from app.models.job import PipelineJob, JobStatus, Domain
+from app.schemas.job import JobCreateResponse, JobDetail, DatasetPreview, ProfileResponse
+from app.services.storage import storage
+from app.services.file_reader import read_dataframe, FileReadError
+from app.services.dataset_inspector import inspect_dataframe
+from app.services.profiling_service import run_profiling_sync
+
+log = get_logger(__name__)
+router = APIRouter(prefix="/datasets", tags=["datasets"])
+
+
+def _extract_extension(filename: str) -> str:
+    if "." not in filename:
+        return ""
+    return filename.rsplit(".", 1)[-1].lower()
+
+
+@router.post(
+    "/upload",
+    response_model=JobCreateResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+async def upload_dataset(
+    file: UploadFile = File(...),
+    domain: str = Form(default="general"),
+    db: Session = Depends(get_db),
+):
+    # ── Validate extension ─────────────────────────────────
+    ext = _extract_extension(file.filename or "")
+    if ext not in settings.allowed_extensions_set:
+        # Be specific for the formats people most often try to upload by mistake.
+        # Generic "use a CSV instead" is unhelpful; tell them which tool to use.
+        format_hint = {
+            "pdf":  "PDFs aren't tabular data. Extract the table first with `pdfplumber` "
+                    "(text PDFs) or `camelot-py` (PDFs with clear table borders), save as CSV, "
+                    "then upload that. Example: `df = pdfplumber.open('f.pdf').pages[0].extract_table()`",
+            "docx": "Word documents aren't tabular. Open in Excel, save as .xlsx or .csv, then upload.",
+            "doc":  "Word documents aren't tabular. Open in Excel, save as .xlsx or .csv, then upload.",
+            "html": "HTML pages aren't directly tabular. Use `pd.read_html('page.html')[0].to_csv(...)` "
+                    "first, then upload the CSV.",
+            "htm":  "HTML pages aren't directly tabular. Use `pd.read_html('page.html')[0].to_csv(...)` "
+                    "first, then upload the CSV.",
+            "txt":  "Plain text isn't reliably tabular. If it's CSV-like, rename to .csv and retry.",
+        }.get(ext, "")
+        msg = f"Unsupported file type '.{ext}'. PrePro Auto handles tabular files: " \
+              f"{sorted(settings.allowed_extensions_set)}."
+        if format_hint:
+            msg += f" {format_hint}"
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=msg)
+
+    # ── Validate domain ────────────────────────────────────
+    try:
+        domain_enum = Domain(domain.lower())
+    except ValueError:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Invalid domain '{domain}'. Use 'general' or 'finance'.",
+        )
+
+    # ── Read bytes and check size ──────────────────────────
+    data = await file.read()
+    size = len(data)
+    if size == 0:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Uploaded file is empty.",
+        )
+    if size > settings.effective_max_upload_bytes:
+        eff = settings.effective_max_upload_mb
+        raise HTTPException(
+            status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
+            detail=(
+                f"File is {size / (1024 * 1024):.0f} MB, which exceeds the current "
+                f"{eff} MB limit. This limit is based on available memory because the "
+                f"dataset is processed in RAM. For larger files, free up memory, run on "
+                f"a machine with more RAM, or pre-sample the data."
+            ),
+        )
+
+    # ── Store raw file ─────────────────────────────────────
+    job_id = str(uuid.uuid4())
+    storage_key = f"raw/{job_id}.{ext}"
+    storage.save_bytes(storage_key, data)
+
+    # ── Create job record ──────────────────────────────────
+    job = PipelineJob(
+        id=job_id,
+        original_filename=file.filename or f"upload.{ext}",
+        file_extension=ext,
+        storage_key=storage_key,
+        file_size_bytes=size,
+        status=JobStatus.UPLOADED,
+        domain=domain_enum,
+    )
+    db.add(job)
+    db.commit()
+    db.refresh(job)
+
+    # If the file is within the streamable limit but too big for the heavy
+    # in-memory steps on this machine, note it (the UI also warns the user).
+    from app.core.memory import fits_heavy_ops
+    heavy = fits_heavy_ops(size)
+    if not heavy["ok"]:
+        log.warning(
+            f"Job {job_id[:8]}: {heavy['file_mb']} MB exceeds heavy-op limit "
+            f"({heavy['heavy_max_mb']} MB) — KNN/MICE/IsolationForest may not fit in RAM."
+        )
+
+    log.info(
+        f"Upload stored: job={job_id[:8]} file={file.filename!r} "
+        f"size={size} domain={domain_enum.value}"
+    )
+
+    # NOTE: In Week 3 we will trigger the profiler Celery task here:
+    #   from app.workers.tasks import profile_dataset
+    #   profile_dataset.delay(job_id)
+
+    return JobCreateResponse(
+        job_id=job.id,
+        original_filename=job.original_filename,
+        file_extension=job.file_extension,
+        file_size_bytes=job.file_size_bytes,
+        status=job.status,
+        domain=job.domain,
+    )
+
+
+@router.get("/{job_id}", response_model=JobDetail)
+def get_dataset(job_id: str, db: Session = Depends(get_db)):
+    job = db.get(PipelineJob, job_id)
+    if job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job {job_id} not found.",
+        )
+    return job
+
+
+@router.get("/{job_id}/preview", response_model=DatasetPreview)
+def preview_dataset(
+    job_id: str,
+    rows: int = 10,
+    db: Session = Depends(get_db),
+):
+    """
+    Load the stored raw file, parse it into a DataFrame, and return its shape
+    plus a preview of the first `rows` rows.
+
+    As a side effect, persists row_count and column_count onto the job record
+    (this is the first time we actually learn the dataset's shape).
+    """
+    job = db.get(PipelineJob, job_id)
+    if job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job {job_id} not found.",
+        )
+
+    # Clamp preview rows to a sane range.
+    rows = max(1, min(rows, 100))
+
+    # Load the raw bytes from storage.
+    try:
+        data = storage.load_bytes(job.storage_key)
+    except Exception as e:  # noqa: BLE001
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Could not load stored file: {e}",
+        )
+
+    # Parse into a DataFrame.
+    try:
+        df = read_dataframe(data, job.file_extension)
+    except FileReadError as e:
+        # Record the failure on the job so the developer can see why.
+        job.status = JobStatus.FAILED
+        job.error_message = str(e)
+        db.commit()
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
+        )
+
+    summary = inspect_dataframe(df, preview_rows=rows)
+
+    # Persist the shape onto the job record.
+    job.row_count = summary["row_count"]
+    job.column_count = summary["column_count"]
+    db.commit()
+
+    log.info(
+        f"Preview generated: job={job_id[:8]} "
+        f"shape=({summary['row_count']}, {summary['column_count']})"
+    )
+
+    return DatasetPreview(job_id=job_id, **summary)
+
+
+@router.post("/{job_id}/profile", response_model=ProfileResponse)
+def profile_dataset_endpoint(
+    job_id: str,
+    background: bool = False,
+    db: Session = Depends(get_db),
+):
+    """
+    Run the statistical profiler on a dataset.
+
+    By default this runs SYNCHRONOUSLY (background=false) — no Redis/Celery
+    needed, perfect for local development. The response includes the full
+    profile.
+
+    Set background=true to dispatch the work to a Celery worker instead (needs
+    Redis + a running worker). In that case the response returns immediately
+    with status=profiling and you poll GET .../profile for the result.
+    """
+    job = db.get(PipelineJob, job_id)
+    if job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job {job_id} not found.",
+        )
+
+    if background:
+        # Dispatch to Celery. Import here so local sync mode never needs Redis.
+        from app.workers.tasks import profile_dataset as profile_task
+        profile_task.delay(job_id)
+        job.status = JobStatus.PROFILING
+        db.commit()
+        return ProfileResponse(
+            job_id=job_id,
+            status=JobStatus.PROFILING,
+            quality_score=None,
+            row_count=job.row_count,
+            column_count=job.column_count,
+        )
+
+    # Synchronous path (default).
+    try:
+        job = run_profiling_sync(job_id, db)
+    except FileReadError as e:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
+        )
+
+    profile = job.profile_json or {}
+    return ProfileResponse(
+        job_id=job_id,
+        status=job.status,
+        quality_score=job.quality_score,
+        row_count=job.row_count,
+        column_count=job.column_count,
+        type_breakdown=profile.get("type_breakdown"),
+        columns=profile.get("columns"),
+    )
+
+
+@router.get("/{job_id}/profile", response_model=ProfileResponse)
+def get_profile(job_id: str, db: Session = Depends(get_db)):
+    """Retrieve a previously-computed profile."""
+    job = db.get(PipelineJob, job_id)
+    if job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job {job_id} not found.",
+        )
+
+    profile = job.profile_json or {}
+    return ProfileResponse(
+        job_id=job_id,
+        status=job.status,
+        quality_score=job.quality_score,
+        row_count=job.row_count,
+        column_count=job.column_count,
+        type_breakdown=profile.get("type_breakdown"),
+        columns=profile.get("columns"),
+    )
+
+
+@router.get("/{job_id}/comparison")
+def dataset_comparison(job_id: str, db: Session = Depends(get_db)):
+    """
+    Before/after comparison for the workbench.
+
+    Returns the original (raw) data's quality and per-column types alongside the
+    cleaned data's quality and types, plus a small sample of each. Lets a user
+    see exactly what preprocessing changed: type conversions, dropped/added
+    columns, and the overall quality-score movement.
+    """
+    import io as _io
+    import pandas as _pd
+    from app.preprocessing.profiler import profile_dataframe
+    from app.models.snapshot import PipelineSnapshot
+
+    job = db.get(PipelineJob, job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail=f"Job {job_id} not found.")
+
+    # ── BEFORE: the raw uploaded file ──────────────────────
+    raw_df = read_dataframe(storage.load_bytes(job.storage_key), job.file_extension)
+    before = profile_dataframe(raw_df, use_llm=False)
+
+    # ── AFTER: the ACTIVE snapshot version (respects undo/redo) ──
+    from app.services.versioning_service import load_active_dataframe
+    active = job.active_version or 0
+    if active > 0:
+        clean_df = load_active_dataframe(job, db)
+        after = profile_dataframe(clean_df, use_llm=False)
+        cleaned = True
+        version = active
+    else:
+        clean_df = raw_df
+        after = before
+        cleaned = False
+        version = 0
+
+    def _types(profile):
+        return {
+            c["name"]: (c.get("refined_type") or c.get("semantic_type", "unknown"))
+            for c in profile["columns"]
+        }
+
+    def _sample(df, n=5):
+        head = df.head(n)
+        return {
+            "columns": [str(c) for c in df.columns],
+            "rows": [
+                {str(c): (None if _pd.isna(r[c]) else _json_cell(r[c])) for c in df.columns}
+                for _, r in head.iterrows()
+            ],
+        }
+
+    return {
+        "job_id": job_id,
+        "cleaned": cleaned,
+        "snapshot_version": version,
+        "before": {
+            "quality_score": before["quality_score"],
+            "row_count": before["row_count"],
+            "column_count": before["column_count"],
+            "types": _types(before),
+            "sample": _sample(raw_df),
+        },
+        "after": {
+            "quality_score": after["quality_score"],
+            "row_count": after["row_count"],
+            "column_count": after["column_count"],
+            "types": _types(after),
+            "sample": _sample(clean_df),
+        },
+    }
+
+
+def _json_cell(v):
+    """Make a single cell JSON-safe (numpy scalars -> python)."""
+    try:
+        import numpy as _np
+        if isinstance(v, (_np.integer,)):
+            return int(v)
+        if isinstance(v, (_np.floating,)):
+            f = float(v)
+            return None if f != f else f  # NaN guard
+        if isinstance(v, (_np.bool_,)):
+            return bool(v)
+    except Exception:
+        pass
+    return v
+
+
+@router.get("/{job_id}/view")
+def view_dataset(
+    job_id: str,
+    rows: int = 25,
+    offset: int = 0,
+    db: Session = Depends(get_db),
+):
+    """
+    Live view of the dataset at the job's ACTIVE version (respects undo/redo).
+    Returns columns (with dtypes), a row sample, shape, and the version label so
+    the UI's View section always reflects the current state after any change.
+
+    Supports windowed paging via `offset`/`rows` so the UI can lazily "Load more"
+    rows in batches instead of rendering the whole frame at once. `total_rows`
+    tells the UI when it has reached the end.
+    """
+    import pandas as _pd
+    from app.services.versioning_service import load_active_dataframe, history as _history
+
+    job = db.get(PipelineJob, job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail=f"Job {job_id} not found.")
+
+    rows = max(1, min(rows, 200))
+    offset = max(0, offset)
+    df = load_active_dataframe(job, db)
+    total_rows = int(df.shape[0])
+    window = df.iloc[offset:offset + rows]
+
+    cols = [{"name": str(c), "dtype": str(df[c].dtype)} for c in df.columns]
+    sample = [
+        {str(c): (None if _pd.isna(r[c]) else _json_cell(r[c])) for c in df.columns}
+        for _, r in window.iterrows()
+    ]
+    h = _history(job, db)
+
+    # Live quality score of the CURRENT version + the raw baseline, so the UI can
+    # show the score and turn green when the data has improved over raw.
+    from app.preprocessing.profiler import profile_dataframe
+    try:
+        current_quality = profile_dataframe(df, use_llm=False)["quality_score"]
+    except Exception:  # noqa: BLE001 - quality is best-effort, never blocks the view
+        current_quality = None
+    baseline_quality = job.quality_score  # set at profile time on the raw data
+
+    returned = offset + len(sample)
+    return {
+        "job_id": job_id,
+        "active_version": h["active_version"],
+        "version_label": next(
+            (v["label"] for v in h["versions"] if v["version"] == h["active_version"]),
+            "Raw data",
+        ),
+        "row_count": total_rows,
+        "total_rows": total_rows,
+        "offset": offset,
+        "returned": returned,
+        "has_more": returned < total_rows,
+        "column_count": int(df.shape[1]),
+        "quality_score": current_quality,
+        "baseline_quality": baseline_quality,
+        "improved": (
+            current_quality is not None and baseline_quality is not None
+            and current_quality > baseline_quality
+        ),
+        "columns": cols,
+        "sample": sample,
+    }
+
+
+@router.get("/{job_id}/history")
+def get_history(job_id: str, db: Session = Depends(get_db)):
+    """Snapshot history + undo/redo availability for the job."""
+    from app.services.versioning_service import history as _history
+    job = db.get(PipelineJob, job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail=f"Job {job_id} not found.")
+    return _history(job, db)
+
+
+@router.post("/{job_id}/undo")
+def undo_step(job_id: str, db: Session = Depends(get_db)):
+    """Move the active version back one step (undo)."""
+    from app.services.versioning_service import undo as _undo
+    job = db.get(PipelineJob, job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail=f"Job {job_id} not found.")
+    return _undo(job, db)
+
+
+@router.post("/{job_id}/redo")
+def redo_step(job_id: str, db: Session = Depends(get_db)):
+    """Move the active version forward one step (redo)."""
+    from app.services.versioning_service import redo as _redo
+    job = db.get(PipelineJob, job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail=f"Job {job_id} not found.")
+    return _redo(job, db)