local-data-studio 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- local_data_studio/__init__.py +3 -0
- local_data_studio/__main__.py +6 -0
- local_data_studio/app.py +150 -0
- local_data_studio/cli.py +154 -0
- local_data_studio/runtime_config.py +77 -0
- local_data_studio/server/__init__.py +1 -0
- local_data_studio/server/api/__init__.py +1 -0
- local_data_studio/server/api/analysis.py +123 -0
- local_data_studio/server/api/atlas_proxy.py +156 -0
- local_data_studio/server/api/datasets.py +176 -0
- local_data_studio/server/api/jobs.py +231 -0
- local_data_studio/server/api/mutations.py +63 -0
- local_data_studio/server/api/schemas.py +117 -0
- local_data_studio/server/api/services.py +99 -0
- local_data_studio/server/api/static.py +34 -0
- local_data_studio/server/atlas.py +91 -0
- local_data_studio/server/atlas_cache.py +20 -0
- local_data_studio/server/atlas_cache_patch.py +40 -0
- local_data_studio/server/atlas_components/__init__.py +1 -0
- local_data_studio/server/atlas_components/command.py +80 -0
- local_data_studio/server/atlas_components/contracts.py +101 -0
- local_data_studio/server/atlas_components/dataset.py +233 -0
- local_data_studio/server/atlas_components/embedding_backends.py +260 -0
- local_data_studio/server/atlas_components/embedding_session.py +89 -0
- local_data_studio/server/atlas_components/image_values.py +193 -0
- local_data_studio/server/atlas_components/images.py +63 -0
- local_data_studio/server/atlas_components/output_frames.py +157 -0
- local_data_studio/server/atlas_components/ports.py +140 -0
- local_data_studio/server/atlas_components/process.py +175 -0
- local_data_studio/server/atlas_components/projection.py +223 -0
- local_data_studio/server/atlas_components/projection_inputs.py +242 -0
- local_data_studio/server/atlas_components/prompts.py +93 -0
- local_data_studio/server/atlas_components/readiness.py +60 -0
- local_data_studio/server/atlas_components/reducers.py +116 -0
- local_data_studio/server/atlas_components/runtime.py +205 -0
- local_data_studio/server/atlas_components/sampling.py +73 -0
- local_data_studio/server/atlas_components/service.py +221 -0
- local_data_studio/server/cache.py +143 -0
- local_data_studio/server/column_stats/__init__.py +35 -0
- local_data_studio/server/column_stats/accumulator.py +162 -0
- local_data_studio/server/column_stats/heuristics.py +187 -0
- local_data_studio/server/column_stats/service.py +52 -0
- local_data_studio/server/config.py +269 -0
- local_data_studio/server/dataset_readers/__init__.py +5 -0
- local_data_studio/server/dataset_readers/common.py +216 -0
- local_data_studio/server/dataset_readers/contracts.py +38 -0
- local_data_studio/server/dataset_readers/delimited.py +174 -0
- local_data_studio/server/dataset_readers/json_reader.py +86 -0
- local_data_studio/server/dataset_readers/jsonl.py +182 -0
- local_data_studio/server/dataset_readers/line.py +39 -0
- local_data_studio/server/dataset_readers/line_cursor.py +75 -0
- local_data_studio/server/dataset_readers/line_indexing.py +57 -0
- local_data_studio/server/dataset_readers/parquet.py +202 -0
- local_data_studio/server/dataset_readers/service.py +228 -0
- local_data_studio/server/db.py +187 -0
- local_data_studio/server/delete_ops.py +261 -0
- local_data_studio/server/deleted_rows.py +30 -0
- local_data_studio/server/eda.py +106 -0
- local_data_studio/server/eda_reports.py +178 -0
- local_data_studio/server/embedder_capabilities.py +384 -0
- local_data_studio/server/embedder_models.py +70 -0
- local_data_studio/server/files.py +190 -0
- local_data_studio/server/jobs.py +284 -0
- local_data_studio/server/line_index.py +126 -0
- local_data_studio/server/llm.py +40 -0
- local_data_studio/server/llm_client.py +91 -0
- local_data_studio/server/llm_profiles.py +297 -0
- local_data_studio/server/llm_prompt.py +81 -0
- local_data_studio/server/llm_service.py +39 -0
- local_data_studio/server/readers.py +29 -0
- local_data_studio/server/serialization.py +118 -0
- local_data_studio/server/sql.py +277 -0
- local_data_studio/server/stats.py +33 -0
- local_data_studio/static/app/application.js +1789 -0
- local_data_studio/static/app/atlas.js +354 -0
- local_data_studio/static/app/dom.js +77 -0
- local_data_studio/static/app/formatting.js +69 -0
- local_data_studio/static/app/http.js +74 -0
- local_data_studio/static/app/images.js +212 -0
- local_data_studio/static/app/llm.js +129 -0
- local_data_studio/static/app/state.js +58 -0
- local_data_studio/static/app.js +1 -0
- local_data_studio/static/index.html +384 -0
- local_data_studio/static/styles.css +1568 -0
- local_data_studio-1.0.0.dist-info/METADATA +683 -0
- local_data_studio-1.0.0.dist-info/RECORD +89 -0
- local_data_studio-1.0.0.dist-info/WHEEL +4 -0
- local_data_studio-1.0.0.dist-info/entry_points.txt +2 -0
- local_data_studio-1.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"""Dataset discovery, upload, preview, and sample endpoints."""
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
import mimetypes
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from fastapi import APIRouter, File, HTTPException, Query, UploadFile
|
|
9
|
+
from fastapi.responses import FileResponse
|
|
10
|
+
|
|
11
|
+
from ..config import (
|
|
12
|
+
ALLOW_DELETE_DATA,
|
|
13
|
+
DATA_ROOT,
|
|
14
|
+
DEFAULT_LIMIT,
|
|
15
|
+
EMBEDDER_MODELS_DIR,
|
|
16
|
+
FILE_SERVE_ROOTS,
|
|
17
|
+
SINGLE_FILE,
|
|
18
|
+
UPLOAD_EXTENSIONS,
|
|
19
|
+
VIS_EXCLUDE_DIRS,
|
|
20
|
+
VIS_EXCLUDE_FILES,
|
|
21
|
+
)
|
|
22
|
+
from ..db import open_connection, quote_ident, relation_sql
|
|
23
|
+
from ..deleted_rows import deleted_row_ids_for
|
|
24
|
+
from ..embedder_capabilities import library_versions
|
|
25
|
+
from ..embedder_models import discover_embedder_models
|
|
26
|
+
from ..files import refresh_dataset_file_catalog, resolve_data_file, resolve_raw_image_file, unique_path
|
|
27
|
+
from ..readers import fetch_preview_page, fetch_raw_row, load_dataset_metadata
|
|
28
|
+
from ..serialization import serialize_raw_value, serialize_value
|
|
29
|
+
from ..sql import fetch_raw_query_row_guarded
|
|
30
|
+
from .schemas import RawRowRequest
|
|
31
|
+
from .services import reject_large_sync_operation
|
|
32
|
+
|
|
33
|
+
router = APIRouter()
|
|
34
|
+
UPLOAD_FILES = File(...)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@router.get("/api/files")
|
|
38
|
+
def list_files() -> dict[str, Any]:
|
|
39
|
+
"""Refresh the dataset allowlist and return file sizes and modification times."""
|
|
40
|
+
files: list[dict[str, Any]] = []
|
|
41
|
+
for name, path in refresh_dataset_file_catalog().items():
|
|
42
|
+
stat = path.stat()
|
|
43
|
+
files.append(
|
|
44
|
+
{
|
|
45
|
+
"name": name,
|
|
46
|
+
"size": stat.st_size,
|
|
47
|
+
"modified": datetime.datetime.fromtimestamp(stat.st_mtime).isoformat(),
|
|
48
|
+
}
|
|
49
|
+
)
|
|
50
|
+
return {"files": files}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@router.get("/api/config")
|
|
54
|
+
def get_config() -> dict[str, Any]:
|
|
55
|
+
"""Return non-secret UI configuration and local file-serving roots."""
|
|
56
|
+
return {
|
|
57
|
+
"allow_delete_data": ALLOW_DELETE_DATA,
|
|
58
|
+
"file_serve_roots": [str(path) for path in FILE_SERVE_ROOTS],
|
|
59
|
+
"vis_exclude_dirs": VIS_EXCLUDE_DIRS,
|
|
60
|
+
"vis_exclude_files": VIS_EXCLUDE_FILES,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@router.get("/api/embedder_models")
|
|
65
|
+
def embedder_models() -> dict[str, Any]:
|
|
66
|
+
"""Discover locally installed encoder model directories."""
|
|
67
|
+
return {"models": discover_embedder_models(EMBEDDER_MODELS_DIR), "libraries": library_versions()}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@router.get("/api/raw")
|
|
71
|
+
def raw_file(path: str = Query(..., description="Absolute file path on the server")) -> FileResponse:
|
|
72
|
+
"""Serve an allowed local image path after root-boundary validation."""
|
|
73
|
+
resolved = resolve_raw_image_file(path, FILE_SERVE_ROOTS)
|
|
74
|
+
media_type, _ = mimetypes.guess_type(str(resolved))
|
|
75
|
+
return FileResponse(str(resolved), media_type=media_type or "application/octet-stream")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@router.post("/api/upload")
|
|
79
|
+
async def upload_files(files: list[UploadFile] = UPLOAD_FILES) -> dict[str, Any]:
|
|
80
|
+
"""Stream supported uploads into the data directory in 1 MiB chunks.
|
|
81
|
+
|
|
82
|
+
Filenames are reduced to their basename and collisions receive a unique suffix.
|
|
83
|
+
Unsupported uploads are closed and reported without being written.
|
|
84
|
+
|
|
85
|
+
Raises:
|
|
86
|
+
HTTPException: Uploads are disabled or no supported file was supplied.
|
|
87
|
+
"""
|
|
88
|
+
if SINGLE_FILE:
|
|
89
|
+
raise HTTPException(status_code=400, detail="uploads are disabled")
|
|
90
|
+
DATA_ROOT.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
saved: list[str] = []
|
|
92
|
+
skipped: list[str] = []
|
|
93
|
+
|
|
94
|
+
for upload in files:
|
|
95
|
+
filename = Path(upload.filename or "").name
|
|
96
|
+
if not filename:
|
|
97
|
+
continue
|
|
98
|
+
if Path(filename).suffix.lower() not in UPLOAD_EXTENSIONS:
|
|
99
|
+
skipped.append(filename)
|
|
100
|
+
await upload.close()
|
|
101
|
+
continue
|
|
102
|
+
destination = unique_path(DATA_ROOT, filename)
|
|
103
|
+
with destination.open("wb") as output:
|
|
104
|
+
while chunk := await upload.read(1024 * 1024):
|
|
105
|
+
output.write(chunk)
|
|
106
|
+
await upload.close()
|
|
107
|
+
saved.append(destination.name)
|
|
108
|
+
|
|
109
|
+
if not saved:
|
|
110
|
+
raise HTTPException(status_code=400, detail="unsupported file extension")
|
|
111
|
+
refresh_dataset_file_catalog()
|
|
112
|
+
return {"saved": saved, "skipped": skipped}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@router.get("/api/schema")
|
|
116
|
+
def get_schema(file: str = Query(...)) -> dict[str, Any]:
|
|
117
|
+
"""Return bounded format-specific metadata for an allowlisted dataset."""
|
|
118
|
+
metadata = load_dataset_metadata(resolve_data_file(file))
|
|
119
|
+
return metadata.to_response(file)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@router.get("/api/preview")
|
|
123
|
+
def preview(
|
|
124
|
+
file: str = Query(...),
|
|
125
|
+
limit: int | None = Query(DEFAULT_LIMIT),
|
|
126
|
+
offset: int | None = Query(0),
|
|
127
|
+
page_token: str | None = Query(None),
|
|
128
|
+
) -> dict[str, Any]:
|
|
129
|
+
"""Return one bounded preview page using a cursor token when supplied."""
|
|
130
|
+
path = resolve_data_file(file)
|
|
131
|
+
return fetch_preview_page(file, path, limit=limit, offset=offset, page_token=page_token, deleted_ids=deleted_row_ids_for(path))
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@router.post("/api/raw_row")
|
|
135
|
+
def raw_row(payload: RawRowRequest) -> dict[str, Any]:
|
|
136
|
+
"""Return an untruncated dataset row or validated SQL result row.
|
|
137
|
+
|
|
138
|
+
Raises:
|
|
139
|
+
HTTPException: Neither a one-based dataset row ID nor SQL was supplied.
|
|
140
|
+
"""
|
|
141
|
+
path = resolve_data_file(payload.file)
|
|
142
|
+
if payload.row_id is not None:
|
|
143
|
+
columns, values = fetch_raw_row(path, payload.row_id)
|
|
144
|
+
row_id = payload.row_id
|
|
145
|
+
elif payload.sql:
|
|
146
|
+
columns, values = fetch_raw_query_row_guarded(path=path, sql=payload.sql, offset=max(0, payload.offset or 0))
|
|
147
|
+
row_id = None
|
|
148
|
+
else:
|
|
149
|
+
raise HTTPException(status_code=400, detail="row_id or sql is required")
|
|
150
|
+
return {
|
|
151
|
+
"file": payload.file,
|
|
152
|
+
"row_id": row_id,
|
|
153
|
+
"columns": columns,
|
|
154
|
+
"row": [serialize_raw_value(value) for value in values],
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@router.get("/api/column_sample")
|
|
159
|
+
def column_sample(
|
|
160
|
+
file: str = Query(...),
|
|
161
|
+
column: str = Query(...),
|
|
162
|
+
limit: int | None = Query(20),
|
|
163
|
+
) -> dict[str, Any]:
|
|
164
|
+
"""Return up to 100 serialized non-null values from one column."""
|
|
165
|
+
path = resolve_data_file(file)
|
|
166
|
+
reject_large_sync_operation(path, "synchronous column sample")
|
|
167
|
+
relation, params = relation_sql(path)
|
|
168
|
+
limit_value = max(1, min(100, limit or 20))
|
|
169
|
+
quoted_column = quote_ident(column)
|
|
170
|
+
with open_connection() as connection:
|
|
171
|
+
sample = connection.execute(
|
|
172
|
+
f"SELECT {quoted_column} FROM {relation} WHERE {quoted_column} IS NOT NULL LIMIT {limit_value}",
|
|
173
|
+
params,
|
|
174
|
+
)
|
|
175
|
+
rows = [serialize_value(row[0]) for row in sample.fetchall()]
|
|
176
|
+
return {"file": file, "column": column, "values": rows}
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
"""Background job API endpoints."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, cast
|
|
4
|
+
|
|
5
|
+
from fastapi import APIRouter, HTTPException, Request
|
|
6
|
+
|
|
7
|
+
from ..atlas_components.runtime import AtlasRuntime
|
|
8
|
+
from ..cache import count_cache_path, search_cache_path
|
|
9
|
+
from ..config import ATLAS_MAX_INSTANCES
|
|
10
|
+
from ..db import normalize_pagination
|
|
11
|
+
from ..deleted_rows import deleted_row_ids_for
|
|
12
|
+
from ..files import resolve_data_file
|
|
13
|
+
from ..jobs import JOB_STORE, JobContext, JobStore
|
|
14
|
+
from ..readers import build_line_index_with_progress, count_rows_with_progress, search_dataset
|
|
15
|
+
from ..sql import execute_query_guarded
|
|
16
|
+
from .schemas import (
|
|
17
|
+
AtlasQueryRequest,
|
|
18
|
+
AtlasRequest,
|
|
19
|
+
CountJobRequest,
|
|
20
|
+
EdaQueryRequest,
|
|
21
|
+
EdaRequest,
|
|
22
|
+
IndexJobRequest,
|
|
23
|
+
QueryRequest,
|
|
24
|
+
SearchJobRequest,
|
|
25
|
+
StatsJobRequest,
|
|
26
|
+
)
|
|
27
|
+
from .services import atlas_service, compute_cached_column_stats, eda_reports_service, load_cached_result, write_cached_result
|
|
28
|
+
|
|
29
|
+
router = APIRouter()
|
|
30
|
+
DEFAULT_ATLAS_RUNTIME = AtlasRuntime(max_instances=ATLAS_MAX_INSTANCES)
|
|
31
|
+
REQUEST_DEFAULT = cast(Request, None)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _job_store(request: Request | None) -> JobStore:
|
|
35
|
+
if request is not None:
|
|
36
|
+
return request.app.state.job_store
|
|
37
|
+
return JOB_STORE
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _atlas_runtime(request: Request | None) -> AtlasRuntime:
|
|
41
|
+
if request is not None:
|
|
42
|
+
return request.app.state.atlas_runtime
|
|
43
|
+
return DEFAULT_ATLAS_RUNTIME
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@router.post("/api/jobs/count")
|
|
47
|
+
def start_count_job(payload: CountJobRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
48
|
+
"""Submit a cancellable row-count job with fingerprinted cache reuse."""
|
|
49
|
+
path = resolve_data_file(payload.file)
|
|
50
|
+
deleted_ids = deleted_row_ids_for(path)
|
|
51
|
+
|
|
52
|
+
def _work(context: JobContext) -> dict[str, Any]:
|
|
53
|
+
cache_path = count_cache_path(path, deleted_ids=deleted_ids)
|
|
54
|
+
cached = load_cached_result(cache_path)
|
|
55
|
+
if cached is not None and isinstance(cached.get("count"), int):
|
|
56
|
+
context.update(progress=1.0, message="Count loaded from cache")
|
|
57
|
+
return {**cached, "cached": True}
|
|
58
|
+
result = {"file": payload.file, "count": count_rows_with_progress(path, context, deleted_ids=deleted_ids)}
|
|
59
|
+
write_cached_result(cache_path, result)
|
|
60
|
+
return {**result, "cached": False}
|
|
61
|
+
|
|
62
|
+
return _job_store(request).submit("count", _work).to_response()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@router.post("/api/jobs/index")
|
|
66
|
+
def start_index_job(payload: IndexJobRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
67
|
+
"""Submit incremental sparse-index construction for a line dataset."""
|
|
68
|
+
path = resolve_data_file(payload.file)
|
|
69
|
+
|
|
70
|
+
def _work(context: JobContext) -> dict[str, Any]:
|
|
71
|
+
return {"file": payload.file, **build_line_index_with_progress(path, context)}
|
|
72
|
+
|
|
73
|
+
return _job_store(request).submit("index", _work).to_response()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@router.post("/api/jobs/search")
|
|
77
|
+
def start_search_job(payload: SearchJobRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
78
|
+
"""Submit a cancellable bounded search with fingerprinted cache reuse."""
|
|
79
|
+
path = resolve_data_file(payload.file)
|
|
80
|
+
deleted_ids = deleted_row_ids_for(path)
|
|
81
|
+
search_term = payload.query.strip()
|
|
82
|
+
limit_value, _ = normalize_pagination(payload.limit, 0)
|
|
83
|
+
|
|
84
|
+
def _work(context: JobContext) -> dict[str, Any]:
|
|
85
|
+
cache_path = search_cache_path(path, query=search_term, limit=limit_value, deleted_ids=deleted_ids)
|
|
86
|
+
cached = load_cached_result(cache_path)
|
|
87
|
+
if cached is not None and cached.get("query") == search_term:
|
|
88
|
+
context.update(progress=1.0, message="Search loaded from cache")
|
|
89
|
+
return {**cached, "cached": True}
|
|
90
|
+
result = search_dataset(payload.file, path, query=search_term, limit=limit_value, control=context, deleted_ids=deleted_ids)
|
|
91
|
+
write_cached_result(cache_path, result)
|
|
92
|
+
return {**result, "cached": False}
|
|
93
|
+
|
|
94
|
+
return _job_store(request).submit("search", _work).to_response()
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@router.post("/api/jobs/query")
|
|
98
|
+
def start_query_job(payload: QueryRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
99
|
+
"""Submit validated read-only SQL under query resource limits."""
|
|
100
|
+
path = resolve_data_file(payload.file)
|
|
101
|
+
|
|
102
|
+
def _work(context: JobContext) -> dict[str, Any]:
|
|
103
|
+
return execute_query_guarded(
|
|
104
|
+
file_name=payload.file,
|
|
105
|
+
path=path,
|
|
106
|
+
sql=payload.sql,
|
|
107
|
+
limit=payload.limit,
|
|
108
|
+
offset=payload.offset,
|
|
109
|
+
context=context,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
return _job_store(request).submit("query", _work).to_response()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@router.post("/api/jobs/stats")
|
|
116
|
+
def start_stats_job(payload: StatsJobRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
117
|
+
"""Submit bounded column sampling with optional cache refresh."""
|
|
118
|
+
path = resolve_data_file(payload.file)
|
|
119
|
+
|
|
120
|
+
def _work(context: JobContext) -> dict[str, Any]:
|
|
121
|
+
context.update(progress=0.15, message="Sampling rows")
|
|
122
|
+
result = compute_cached_column_stats(payload.file, path, payload.sample, bool(payload.force))
|
|
123
|
+
context.update(progress=1.0, message="Stats ready")
|
|
124
|
+
return result
|
|
125
|
+
|
|
126
|
+
return _job_store(request).submit("stats", _work).to_response()
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@router.post("/api/jobs/eda")
|
|
130
|
+
def start_eda_job(payload: EdaRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
131
|
+
"""Submit EDA generation for a bounded dataset sample."""
|
|
132
|
+
path = resolve_data_file(payload.file)
|
|
133
|
+
reports = eda_reports_service()
|
|
134
|
+
|
|
135
|
+
def _work(context: JobContext) -> dict[str, Any]:
|
|
136
|
+
return reports.generate_dataset_eda_report(
|
|
137
|
+
file_name=payload.file,
|
|
138
|
+
path=path,
|
|
139
|
+
sample=payload.sample,
|
|
140
|
+
mode=payload.mode,
|
|
141
|
+
force=payload.force,
|
|
142
|
+
context=context,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
return _job_store(request).submit("eda", _work).to_response()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@router.post("/api/jobs/eda_query")
|
|
149
|
+
def start_eda_query_job(payload: EdaQueryRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
150
|
+
"""Submit EDA generation for validated SQL query results."""
|
|
151
|
+
path = resolve_data_file(payload.file)
|
|
152
|
+
reports = eda_reports_service()
|
|
153
|
+
|
|
154
|
+
def _work(context: JobContext) -> dict[str, Any]:
|
|
155
|
+
return reports.generate_query_eda_report(
|
|
156
|
+
file_name=payload.file,
|
|
157
|
+
path=path,
|
|
158
|
+
sql=payload.sql,
|
|
159
|
+
sample=payload.sample,
|
|
160
|
+
mode=payload.mode,
|
|
161
|
+
force=payload.force,
|
|
162
|
+
context=context,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
return _job_store(request).submit("eda_query", _work).to_response()
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@router.post("/api/jobs/atlas")
|
|
169
|
+
def start_atlas_job(payload: AtlasRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
170
|
+
"""Submit embedding and Atlas startup for a dataset column."""
|
|
171
|
+
path = resolve_data_file(payload.file)
|
|
172
|
+
atlas = atlas_service()
|
|
173
|
+
|
|
174
|
+
def _work(context: JobContext) -> dict[str, Any]:
|
|
175
|
+
return atlas.run_atlas_visualization(
|
|
176
|
+
file_name=payload.file,
|
|
177
|
+
path=path,
|
|
178
|
+
column=payload.column,
|
|
179
|
+
model=payload.model,
|
|
180
|
+
backend=payload.backend,
|
|
181
|
+
prompt=payload.prompt,
|
|
182
|
+
projection_method=payload.projection_method,
|
|
183
|
+
sql=None,
|
|
184
|
+
sample=payload.sample,
|
|
185
|
+
context=context,
|
|
186
|
+
runtime=_atlas_runtime(request),
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
return _job_store(request).submit("atlas", _work).to_response()
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@router.post("/api/jobs/atlas_query")
|
|
193
|
+
def start_atlas_query_job(payload: AtlasQueryRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
194
|
+
"""Submit embedding and Atlas startup for validated SQL results."""
|
|
195
|
+
path = resolve_data_file(payload.file)
|
|
196
|
+
atlas = atlas_service()
|
|
197
|
+
|
|
198
|
+
def _work(context: JobContext) -> dict[str, Any]:
|
|
199
|
+
return atlas.run_atlas_visualization(
|
|
200
|
+
file_name=payload.file,
|
|
201
|
+
path=path,
|
|
202
|
+
column=payload.column,
|
|
203
|
+
model=payload.model,
|
|
204
|
+
backend=payload.backend,
|
|
205
|
+
prompt=payload.prompt,
|
|
206
|
+
projection_method=payload.projection_method,
|
|
207
|
+
sql=payload.sql,
|
|
208
|
+
sample=payload.sample,
|
|
209
|
+
context=context,
|
|
210
|
+
runtime=_atlas_runtime(request),
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
return _job_store(request).submit("atlas_query", _work).to_response()
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@router.get("/api/jobs/{job_id}")
|
|
217
|
+
def get_job(job_id: str, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
218
|
+
"""Return a detached snapshot of a job, or HTTP 404 when unknown."""
|
|
219
|
+
record = _job_store(request).get(job_id)
|
|
220
|
+
if record is None:
|
|
221
|
+
raise HTTPException(status_code=404, detail="job not found")
|
|
222
|
+
return record.to_response()
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
@router.post("/api/jobs/{job_id}/cancel")
|
|
226
|
+
def cancel_job(job_id: str, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
|
|
227
|
+
"""Request cooperative cancellation, or return HTTP 404 when unknown."""
|
|
228
|
+
record = _job_store(request).cancel(job_id)
|
|
229
|
+
if record is None:
|
|
230
|
+
raise HTTPException(status_code=404, detail="job not found")
|
|
231
|
+
return record.to_response()
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Dataset row and column mutation endpoints."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from fastapi import APIRouter, HTTPException
|
|
7
|
+
|
|
8
|
+
from ..config import ALLOW_DELETE_DATA, HARD_DELETE_MAX_BYTES
|
|
9
|
+
from ..db import count_relation_rows_raw
|
|
10
|
+
from ..delete_ops import delete_column_from_file, delete_row_from_file
|
|
11
|
+
from ..deleted_rows import add_deleted_row_id, clear_deleted_row_ids, deleted_row_ids_for
|
|
12
|
+
from ..files import resolve_data_file
|
|
13
|
+
from .schemas import DeleteColumnRequest, DeleteRowRequest
|
|
14
|
+
|
|
15
|
+
router = APIRouter()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _reject_large_hard_delete(path: Path) -> None:
|
|
19
|
+
if path.stat().st_size > HARD_DELETE_MAX_BYTES:
|
|
20
|
+
raise HTTPException(
|
|
21
|
+
status_code=400,
|
|
22
|
+
detail="delete from file rewrites the dataset and is disabled for large files; use hide only",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@router.post("/api/delete_row")
|
|
27
|
+
def delete_row(payload: DeleteRowRequest) -> dict[str, Any]:
|
|
28
|
+
"""Hide a row or rewrite a small dataset when persistent deletion is allowed.
|
|
29
|
+
|
|
30
|
+
Row IDs are one-based. Persistent deletion clears all soft-deleted row IDs.
|
|
31
|
+
"""
|
|
32
|
+
path = resolve_data_file(payload.file)
|
|
33
|
+
row_id = int(payload.row_id)
|
|
34
|
+
if row_id < 1:
|
|
35
|
+
raise HTTPException(status_code=400, detail="row_id must be positive")
|
|
36
|
+
if payload.persist:
|
|
37
|
+
if not ALLOW_DELETE_DATA:
|
|
38
|
+
raise HTTPException(status_code=403, detail="delete from file is disabled")
|
|
39
|
+
_reject_large_hard_delete(path)
|
|
40
|
+
total_rows = count_relation_rows_raw(path)
|
|
41
|
+
if row_id > total_rows:
|
|
42
|
+
raise HTTPException(status_code=404, detail="row not found")
|
|
43
|
+
delete_row_from_file(path, row_id)
|
|
44
|
+
clear_deleted_row_ids(path)
|
|
45
|
+
return {"file": payload.file, "row_id": row_id, "persisted": True, "total_rows": max(0, total_rows - 1)}
|
|
46
|
+
add_deleted_row_id(path, row_id)
|
|
47
|
+
return {"file": payload.file, "row_id": row_id, "persisted": False, "deleted_count": len(deleted_row_ids_for(path))}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@router.post("/api/delete_column")
|
|
51
|
+
def delete_column(payload: DeleteColumnRequest) -> dict[str, Any]:
|
|
52
|
+
"""Acknowledge a soft column hide or persistently rewrite a small dataset."""
|
|
53
|
+
path = resolve_data_file(payload.file)
|
|
54
|
+
column = payload.column.strip()
|
|
55
|
+
if not column:
|
|
56
|
+
raise HTTPException(status_code=400, detail="column is required")
|
|
57
|
+
if payload.persist:
|
|
58
|
+
if not ALLOW_DELETE_DATA:
|
|
59
|
+
raise HTTPException(status_code=403, detail="delete from file is disabled")
|
|
60
|
+
_reject_large_hard_delete(path)
|
|
61
|
+
delete_column_from_file(path, column)
|
|
62
|
+
return {"file": payload.file, "column": column, "persisted": True}
|
|
63
|
+
return {"file": payload.file, "column": column, "persisted": False}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Request schemas shared by Local Data Studio API routers."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Literal
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, Field, field_validator
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class QueryRequest(BaseModel):
|
|
9
|
+
"""Input for a guarded read-only SQL query."""
|
|
10
|
+
|
|
11
|
+
file: str = Field(min_length=1)
|
|
12
|
+
sql: str = Field(min_length=1)
|
|
13
|
+
limit: int | None = Field(default=None, ge=1)
|
|
14
|
+
offset: int | None = Field(default=None, ge=0)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class RawRowRequest(BaseModel):
|
|
18
|
+
"""Input selecting either a one-based dataset row or SQL result offset."""
|
|
19
|
+
|
|
20
|
+
file: str = Field(min_length=1)
|
|
21
|
+
row_id: int | None = Field(default=None, ge=1)
|
|
22
|
+
sql: str | None = None
|
|
23
|
+
offset: int | None = Field(default=None, ge=0)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class EdaRequest(BaseModel):
|
|
27
|
+
"""Input controlling bounded EDA generation and report cache reuse."""
|
|
28
|
+
|
|
29
|
+
file: str = Field(min_length=1)
|
|
30
|
+
sample: int | None = None
|
|
31
|
+
force: bool | None = None
|
|
32
|
+
mode: Literal["minimal", "maximal"] | None = None
|
|
33
|
+
|
|
34
|
+
@field_validator("sample")
|
|
35
|
+
@classmethod
|
|
36
|
+
def validate_sample(cls, value: int | None) -> int | None:
|
|
37
|
+
"""Accept the unlimited marker or a positive row limit."""
|
|
38
|
+
if value is None or value == -1 or value >= 1:
|
|
39
|
+
return value
|
|
40
|
+
raise ValueError("sample must be -1 or an integer greater than or equal to 1")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class EdaQueryRequest(EdaRequest):
|
|
44
|
+
"""EDA input extended with the guarded SQL that supplies report rows."""
|
|
45
|
+
|
|
46
|
+
sql: str = Field(min_length=1)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class AtlasRequest(BaseModel):
|
|
50
|
+
"""Input selecting a dataset column and local model for Atlas embedding."""
|
|
51
|
+
|
|
52
|
+
file: str = Field(min_length=1)
|
|
53
|
+
column: str = Field(min_length=1)
|
|
54
|
+
model: str = Field(min_length=1)
|
|
55
|
+
sample: int | None = Field(default=None, ge=0)
|
|
56
|
+
backend: Literal["transformers", "sentence-transformers"] | None = None
|
|
57
|
+
prompt: str | None = Field(default=None, max_length=16_384)
|
|
58
|
+
projection_method: Literal["umap", "tsne", "pca"] = "umap"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class AtlasQueryRequest(AtlasRequest):
|
|
62
|
+
"""Atlas input extended with the guarded SQL that supplies embedding rows."""
|
|
63
|
+
|
|
64
|
+
sql: str = Field(min_length=1)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class CountJobRequest(BaseModel):
|
|
68
|
+
"""Input for a background row-count operation."""
|
|
69
|
+
|
|
70
|
+
file: str = Field(min_length=1)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class IndexJobRequest(BaseModel):
|
|
74
|
+
"""Input for incremental sparse-index construction."""
|
|
75
|
+
|
|
76
|
+
file: str = Field(min_length=1)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class SearchJobRequest(BaseModel):
|
|
80
|
+
"""Input for a bounded background dataset search."""
|
|
81
|
+
|
|
82
|
+
file: str = Field(min_length=1)
|
|
83
|
+
query: str = Field(min_length=1)
|
|
84
|
+
limit: int | None = Field(default=None, ge=1)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class StatsJobRequest(BaseModel):
|
|
88
|
+
"""Input controlling sampled statistics and cache refresh."""
|
|
89
|
+
|
|
90
|
+
file: str = Field(min_length=1)
|
|
91
|
+
sample: int | None = None
|
|
92
|
+
force: bool | None = None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class DeleteRowRequest(BaseModel):
|
|
96
|
+
"""Input for one-based soft or persistent row deletion."""
|
|
97
|
+
|
|
98
|
+
file: str = Field(min_length=1)
|
|
99
|
+
row_id: int = Field(ge=1)
|
|
100
|
+
persist: bool | None = None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class DeleteColumnRequest(BaseModel):
|
|
104
|
+
"""Input for soft or persistent column deletion."""
|
|
105
|
+
|
|
106
|
+
file: str = Field(min_length=1)
|
|
107
|
+
column: str = Field(min_length=1)
|
|
108
|
+
persist: bool | None = None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class NLQueryRequest(BaseModel):
|
|
112
|
+
"""Input for natural-language SQL generation with optional sample context."""
|
|
113
|
+
|
|
114
|
+
file: str = Field(min_length=1)
|
|
115
|
+
prompt: str = Field(min_length=1, max_length=16_384)
|
|
116
|
+
sample: dict[str, Any] | None = None
|
|
117
|
+
model: str | None = Field(default=None, max_length=64)
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Small shared services used by API routers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import tempfile
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from fastapi import HTTPException
|
|
12
|
+
|
|
13
|
+
from ..cache import stats_cache_path
|
|
14
|
+
from ..config import DEFAULT_SAMPLE
|
|
15
|
+
from ..sql import is_large_dataset
|
|
16
|
+
from ..stats import compute_column_stats
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def atlas_service() -> Any:
|
|
20
|
+
"""Import the optional ML stack only when an Atlas job is requested."""
|
|
21
|
+
from .. import atlas # noqa: PLC0415
|
|
22
|
+
|
|
23
|
+
return atlas
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def eda_reports_service() -> Any:
|
|
27
|
+
"""Import profiling dependencies only when an EDA operation is requested."""
|
|
28
|
+
try:
|
|
29
|
+
from .. import eda_reports # noqa: PLC0415
|
|
30
|
+
except ImportError as exc:
|
|
31
|
+
raise HTTPException(status_code=500, detail=f"EDA dependencies could not be loaded: {exc}") from exc
|
|
32
|
+
return eda_reports
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def reject_large_sync_operation(path: Path, operation: str) -> None:
|
|
36
|
+
"""Reject potentially blocking full scans above the large-dataset threshold."""
|
|
37
|
+
if is_large_dataset(path):
|
|
38
|
+
raise HTTPException(
|
|
39
|
+
status_code=400,
|
|
40
|
+
detail=f"{operation} can scan the full dataset; use the background job endpoint for large files",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def load_cached_result(cache_path: Path) -> dict[str, Any] | None:
|
|
45
|
+
"""Load an object-shaped JSON cache, treating missing or invalid data as a miss."""
|
|
46
|
+
if not cache_path.exists():
|
|
47
|
+
return None
|
|
48
|
+
try:
|
|
49
|
+
payload = json.loads(cache_path.read_text(encoding="utf-8"))
|
|
50
|
+
except (OSError, json.JSONDecodeError):
|
|
51
|
+
return None
|
|
52
|
+
return payload if isinstance(payload, dict) else None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def write_cached_result(cache_path: Path, result: dict[str, Any]) -> None:
|
|
56
|
+
"""Atomically replace a JSON cache file with the supplied result.
|
|
57
|
+
|
|
58
|
+
The caller retains ownership of ``result``; this function does not mutate it.
|
|
59
|
+
"""
|
|
60
|
+
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
|
61
|
+
temporary_path: Path | None = None
|
|
62
|
+
try:
|
|
63
|
+
with tempfile.NamedTemporaryFile(
|
|
64
|
+
mode="w",
|
|
65
|
+
encoding="utf-8",
|
|
66
|
+
dir=cache_path.parent,
|
|
67
|
+
prefix=f".{cache_path.name}.",
|
|
68
|
+
suffix=".tmp",
|
|
69
|
+
delete=False,
|
|
70
|
+
) as temporary:
|
|
71
|
+
json.dump(result, temporary, ensure_ascii=False, indent=2)
|
|
72
|
+
temporary.flush()
|
|
73
|
+
os.fsync(temporary.fileno())
|
|
74
|
+
temporary_path = Path(temporary.name)
|
|
75
|
+
os.replace(temporary_path, cache_path)
|
|
76
|
+
temporary_path = None
|
|
77
|
+
finally:
|
|
78
|
+
if temporary_path is not None:
|
|
79
|
+
temporary_path.unlink(missing_ok=True)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _load_cached_stats(path: Path, sample: int) -> dict[str, Any] | None:
|
|
83
|
+
payload = load_cached_result(stats_cache_path(path))
|
|
84
|
+
if payload is None or payload.get("sample_request") != sample:
|
|
85
|
+
return None
|
|
86
|
+
result = payload.get("result")
|
|
87
|
+
return result if isinstance(result, dict) else None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def compute_cached_column_stats(file_name: str, path: Path, sample: int | None, force: bool) -> dict[str, Any]:
|
|
91
|
+
"""Compute or reuse statistics keyed by dataset fingerprint and sample size."""
|
|
92
|
+
sample_value = sample if sample is not None else DEFAULT_SAMPLE
|
|
93
|
+
if not force:
|
|
94
|
+
cached = _load_cached_stats(path, sample_value)
|
|
95
|
+
if cached is not None:
|
|
96
|
+
return {**cached, "cached": True}
|
|
97
|
+
result = compute_column_stats(file_name, path, sample_value)
|
|
98
|
+
write_cached_result(stats_cache_path(path), {"sample_request": sample_value, "result": result})
|
|
99
|
+
return {**result, "cached": False}
|