local-data-studio 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. local_data_studio/__init__.py +3 -0
  2. local_data_studio/__main__.py +6 -0
  3. local_data_studio/app.py +150 -0
  4. local_data_studio/cli.py +154 -0
  5. local_data_studio/runtime_config.py +77 -0
  6. local_data_studio/server/__init__.py +1 -0
  7. local_data_studio/server/api/__init__.py +1 -0
  8. local_data_studio/server/api/analysis.py +123 -0
  9. local_data_studio/server/api/atlas_proxy.py +156 -0
  10. local_data_studio/server/api/datasets.py +176 -0
  11. local_data_studio/server/api/jobs.py +231 -0
  12. local_data_studio/server/api/mutations.py +63 -0
  13. local_data_studio/server/api/schemas.py +117 -0
  14. local_data_studio/server/api/services.py +99 -0
  15. local_data_studio/server/api/static.py +34 -0
  16. local_data_studio/server/atlas.py +91 -0
  17. local_data_studio/server/atlas_cache.py +20 -0
  18. local_data_studio/server/atlas_cache_patch.py +40 -0
  19. local_data_studio/server/atlas_components/__init__.py +1 -0
  20. local_data_studio/server/atlas_components/command.py +80 -0
  21. local_data_studio/server/atlas_components/contracts.py +101 -0
  22. local_data_studio/server/atlas_components/dataset.py +233 -0
  23. local_data_studio/server/atlas_components/embedding_backends.py +260 -0
  24. local_data_studio/server/atlas_components/embedding_session.py +89 -0
  25. local_data_studio/server/atlas_components/image_values.py +193 -0
  26. local_data_studio/server/atlas_components/images.py +63 -0
  27. local_data_studio/server/atlas_components/output_frames.py +157 -0
  28. local_data_studio/server/atlas_components/ports.py +140 -0
  29. local_data_studio/server/atlas_components/process.py +175 -0
  30. local_data_studio/server/atlas_components/projection.py +223 -0
  31. local_data_studio/server/atlas_components/projection_inputs.py +242 -0
  32. local_data_studio/server/atlas_components/prompts.py +93 -0
  33. local_data_studio/server/atlas_components/readiness.py +60 -0
  34. local_data_studio/server/atlas_components/reducers.py +116 -0
  35. local_data_studio/server/atlas_components/runtime.py +205 -0
  36. local_data_studio/server/atlas_components/sampling.py +73 -0
  37. local_data_studio/server/atlas_components/service.py +221 -0
  38. local_data_studio/server/cache.py +143 -0
  39. local_data_studio/server/column_stats/__init__.py +35 -0
  40. local_data_studio/server/column_stats/accumulator.py +162 -0
  41. local_data_studio/server/column_stats/heuristics.py +187 -0
  42. local_data_studio/server/column_stats/service.py +52 -0
  43. local_data_studio/server/config.py +269 -0
  44. local_data_studio/server/dataset_readers/__init__.py +5 -0
  45. local_data_studio/server/dataset_readers/common.py +216 -0
  46. local_data_studio/server/dataset_readers/contracts.py +38 -0
  47. local_data_studio/server/dataset_readers/delimited.py +174 -0
  48. local_data_studio/server/dataset_readers/json_reader.py +86 -0
  49. local_data_studio/server/dataset_readers/jsonl.py +182 -0
  50. local_data_studio/server/dataset_readers/line.py +39 -0
  51. local_data_studio/server/dataset_readers/line_cursor.py +75 -0
  52. local_data_studio/server/dataset_readers/line_indexing.py +57 -0
  53. local_data_studio/server/dataset_readers/parquet.py +202 -0
  54. local_data_studio/server/dataset_readers/service.py +228 -0
  55. local_data_studio/server/db.py +187 -0
  56. local_data_studio/server/delete_ops.py +261 -0
  57. local_data_studio/server/deleted_rows.py +30 -0
  58. local_data_studio/server/eda.py +106 -0
  59. local_data_studio/server/eda_reports.py +178 -0
  60. local_data_studio/server/embedder_capabilities.py +384 -0
  61. local_data_studio/server/embedder_models.py +70 -0
  62. local_data_studio/server/files.py +190 -0
  63. local_data_studio/server/jobs.py +284 -0
  64. local_data_studio/server/line_index.py +126 -0
  65. local_data_studio/server/llm.py +40 -0
  66. local_data_studio/server/llm_client.py +91 -0
  67. local_data_studio/server/llm_profiles.py +297 -0
  68. local_data_studio/server/llm_prompt.py +81 -0
  69. local_data_studio/server/llm_service.py +39 -0
  70. local_data_studio/server/readers.py +29 -0
  71. local_data_studio/server/serialization.py +118 -0
  72. local_data_studio/server/sql.py +277 -0
  73. local_data_studio/server/stats.py +33 -0
  74. local_data_studio/static/app/application.js +1789 -0
  75. local_data_studio/static/app/atlas.js +354 -0
  76. local_data_studio/static/app/dom.js +77 -0
  77. local_data_studio/static/app/formatting.js +69 -0
  78. local_data_studio/static/app/http.js +74 -0
  79. local_data_studio/static/app/images.js +212 -0
  80. local_data_studio/static/app/llm.js +129 -0
  81. local_data_studio/static/app/state.js +58 -0
  82. local_data_studio/static/app.js +1 -0
  83. local_data_studio/static/index.html +384 -0
  84. local_data_studio/static/styles.css +1568 -0
  85. local_data_studio-1.0.0.dist-info/METADATA +683 -0
  86. local_data_studio-1.0.0.dist-info/RECORD +89 -0
  87. local_data_studio-1.0.0.dist-info/WHEEL +4 -0
  88. local_data_studio-1.0.0.dist-info/entry_points.txt +2 -0
  89. local_data_studio-1.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,176 @@
1
+ """Dataset discovery, upload, preview, and sample endpoints."""
2
+
3
+ import datetime
4
+ import mimetypes
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ from fastapi import APIRouter, File, HTTPException, Query, UploadFile
9
+ from fastapi.responses import FileResponse
10
+
11
+ from ..config import (
12
+ ALLOW_DELETE_DATA,
13
+ DATA_ROOT,
14
+ DEFAULT_LIMIT,
15
+ EMBEDDER_MODELS_DIR,
16
+ FILE_SERVE_ROOTS,
17
+ SINGLE_FILE,
18
+ UPLOAD_EXTENSIONS,
19
+ VIS_EXCLUDE_DIRS,
20
+ VIS_EXCLUDE_FILES,
21
+ )
22
+ from ..db import open_connection, quote_ident, relation_sql
23
+ from ..deleted_rows import deleted_row_ids_for
24
+ from ..embedder_capabilities import library_versions
25
+ from ..embedder_models import discover_embedder_models
26
+ from ..files import refresh_dataset_file_catalog, resolve_data_file, resolve_raw_image_file, unique_path
27
+ from ..readers import fetch_preview_page, fetch_raw_row, load_dataset_metadata
28
+ from ..serialization import serialize_raw_value, serialize_value
29
+ from ..sql import fetch_raw_query_row_guarded
30
+ from .schemas import RawRowRequest
31
+ from .services import reject_large_sync_operation
32
+
33
+ router = APIRouter()
34
+ UPLOAD_FILES = File(...)
35
+
36
+
37
+ @router.get("/api/files")
38
+ def list_files() -> dict[str, Any]:
39
+ """Refresh the dataset allowlist and return file sizes and modification times."""
40
+ files: list[dict[str, Any]] = []
41
+ for name, path in refresh_dataset_file_catalog().items():
42
+ stat = path.stat()
43
+ files.append(
44
+ {
45
+ "name": name,
46
+ "size": stat.st_size,
47
+ "modified": datetime.datetime.fromtimestamp(stat.st_mtime).isoformat(),
48
+ }
49
+ )
50
+ return {"files": files}
51
+
52
+
53
+ @router.get("/api/config")
54
+ def get_config() -> dict[str, Any]:
55
+ """Return non-secret UI configuration and local file-serving roots."""
56
+ return {
57
+ "allow_delete_data": ALLOW_DELETE_DATA,
58
+ "file_serve_roots": [str(path) for path in FILE_SERVE_ROOTS],
59
+ "vis_exclude_dirs": VIS_EXCLUDE_DIRS,
60
+ "vis_exclude_files": VIS_EXCLUDE_FILES,
61
+ }
62
+
63
+
64
+ @router.get("/api/embedder_models")
65
+ def embedder_models() -> dict[str, Any]:
66
+ """Discover locally installed encoder model directories."""
67
+ return {"models": discover_embedder_models(EMBEDDER_MODELS_DIR), "libraries": library_versions()}
68
+
69
+
70
+ @router.get("/api/raw")
71
+ def raw_file(path: str = Query(..., description="Absolute file path on the server")) -> FileResponse:
72
+ """Serve an allowed local image path after root-boundary validation."""
73
+ resolved = resolve_raw_image_file(path, FILE_SERVE_ROOTS)
74
+ media_type, _ = mimetypes.guess_type(str(resolved))
75
+ return FileResponse(str(resolved), media_type=media_type or "application/octet-stream")
76
+
77
+
78
+ @router.post("/api/upload")
79
+ async def upload_files(files: list[UploadFile] = UPLOAD_FILES) -> dict[str, Any]:
80
+ """Stream supported uploads into the data directory in 1 MiB chunks.
81
+
82
+ Filenames are reduced to their basename and collisions receive a unique suffix.
83
+ Unsupported uploads are closed and reported without being written.
84
+
85
+ Raises:
86
+ HTTPException: Uploads are disabled or no supported file was supplied.
87
+ """
88
+ if SINGLE_FILE:
89
+ raise HTTPException(status_code=400, detail="uploads are disabled")
90
+ DATA_ROOT.mkdir(parents=True, exist_ok=True)
91
+ saved: list[str] = []
92
+ skipped: list[str] = []
93
+
94
+ for upload in files:
95
+ filename = Path(upload.filename or "").name
96
+ if not filename:
97
+ continue
98
+ if Path(filename).suffix.lower() not in UPLOAD_EXTENSIONS:
99
+ skipped.append(filename)
100
+ await upload.close()
101
+ continue
102
+ destination = unique_path(DATA_ROOT, filename)
103
+ with destination.open("wb") as output:
104
+ while chunk := await upload.read(1024 * 1024):
105
+ output.write(chunk)
106
+ await upload.close()
107
+ saved.append(destination.name)
108
+
109
+ if not saved:
110
+ raise HTTPException(status_code=400, detail="unsupported file extension")
111
+ refresh_dataset_file_catalog()
112
+ return {"saved": saved, "skipped": skipped}
113
+
114
+
115
+ @router.get("/api/schema")
116
+ def get_schema(file: str = Query(...)) -> dict[str, Any]:
117
+ """Return bounded format-specific metadata for an allowlisted dataset."""
118
+ metadata = load_dataset_metadata(resolve_data_file(file))
119
+ return metadata.to_response(file)
120
+
121
+
122
+ @router.get("/api/preview")
123
+ def preview(
124
+ file: str = Query(...),
125
+ limit: int | None = Query(DEFAULT_LIMIT),
126
+ offset: int | None = Query(0),
127
+ page_token: str | None = Query(None),
128
+ ) -> dict[str, Any]:
129
+ """Return one bounded preview page using a cursor token when supplied."""
130
+ path = resolve_data_file(file)
131
+ return fetch_preview_page(file, path, limit=limit, offset=offset, page_token=page_token, deleted_ids=deleted_row_ids_for(path))
132
+
133
+
134
+ @router.post("/api/raw_row")
135
+ def raw_row(payload: RawRowRequest) -> dict[str, Any]:
136
+ """Return an untruncated dataset row or validated SQL result row.
137
+
138
+ Raises:
139
+ HTTPException: Neither a one-based dataset row ID nor SQL was supplied.
140
+ """
141
+ path = resolve_data_file(payload.file)
142
+ if payload.row_id is not None:
143
+ columns, values = fetch_raw_row(path, payload.row_id)
144
+ row_id = payload.row_id
145
+ elif payload.sql:
146
+ columns, values = fetch_raw_query_row_guarded(path=path, sql=payload.sql, offset=max(0, payload.offset or 0))
147
+ row_id = None
148
+ else:
149
+ raise HTTPException(status_code=400, detail="row_id or sql is required")
150
+ return {
151
+ "file": payload.file,
152
+ "row_id": row_id,
153
+ "columns": columns,
154
+ "row": [serialize_raw_value(value) for value in values],
155
+ }
156
+
157
+
158
+ @router.get("/api/column_sample")
159
+ def column_sample(
160
+ file: str = Query(...),
161
+ column: str = Query(...),
162
+ limit: int | None = Query(20),
163
+ ) -> dict[str, Any]:
164
+ """Return up to 100 serialized non-null values from one column."""
165
+ path = resolve_data_file(file)
166
+ reject_large_sync_operation(path, "synchronous column sample")
167
+ relation, params = relation_sql(path)
168
+ limit_value = max(1, min(100, limit or 20))
169
+ quoted_column = quote_ident(column)
170
+ with open_connection() as connection:
171
+ sample = connection.execute(
172
+ f"SELECT {quoted_column} FROM {relation} WHERE {quoted_column} IS NOT NULL LIMIT {limit_value}",
173
+ params,
174
+ )
175
+ rows = [serialize_value(row[0]) for row in sample.fetchall()]
176
+ return {"file": file, "column": column, "values": rows}
@@ -0,0 +1,231 @@
1
+ """Background job API endpoints."""
2
+
3
+ from typing import Any, cast
4
+
5
+ from fastapi import APIRouter, HTTPException, Request
6
+
7
+ from ..atlas_components.runtime import AtlasRuntime
8
+ from ..cache import count_cache_path, search_cache_path
9
+ from ..config import ATLAS_MAX_INSTANCES
10
+ from ..db import normalize_pagination
11
+ from ..deleted_rows import deleted_row_ids_for
12
+ from ..files import resolve_data_file
13
+ from ..jobs import JOB_STORE, JobContext, JobStore
14
+ from ..readers import build_line_index_with_progress, count_rows_with_progress, search_dataset
15
+ from ..sql import execute_query_guarded
16
+ from .schemas import (
17
+ AtlasQueryRequest,
18
+ AtlasRequest,
19
+ CountJobRequest,
20
+ EdaQueryRequest,
21
+ EdaRequest,
22
+ IndexJobRequest,
23
+ QueryRequest,
24
+ SearchJobRequest,
25
+ StatsJobRequest,
26
+ )
27
+ from .services import atlas_service, compute_cached_column_stats, eda_reports_service, load_cached_result, write_cached_result
28
+
29
+ router = APIRouter()
30
+ DEFAULT_ATLAS_RUNTIME = AtlasRuntime(max_instances=ATLAS_MAX_INSTANCES)
31
+ REQUEST_DEFAULT = cast(Request, None)
32
+
33
+
34
+ def _job_store(request: Request | None) -> JobStore:
35
+ if request is not None:
36
+ return request.app.state.job_store
37
+ return JOB_STORE
38
+
39
+
40
+ def _atlas_runtime(request: Request | None) -> AtlasRuntime:
41
+ if request is not None:
42
+ return request.app.state.atlas_runtime
43
+ return DEFAULT_ATLAS_RUNTIME
44
+
45
+
46
+ @router.post("/api/jobs/count")
47
+ def start_count_job(payload: CountJobRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
48
+ """Submit a cancellable row-count job with fingerprinted cache reuse."""
49
+ path = resolve_data_file(payload.file)
50
+ deleted_ids = deleted_row_ids_for(path)
51
+
52
+ def _work(context: JobContext) -> dict[str, Any]:
53
+ cache_path = count_cache_path(path, deleted_ids=deleted_ids)
54
+ cached = load_cached_result(cache_path)
55
+ if cached is not None and isinstance(cached.get("count"), int):
56
+ context.update(progress=1.0, message="Count loaded from cache")
57
+ return {**cached, "cached": True}
58
+ result = {"file": payload.file, "count": count_rows_with_progress(path, context, deleted_ids=deleted_ids)}
59
+ write_cached_result(cache_path, result)
60
+ return {**result, "cached": False}
61
+
62
+ return _job_store(request).submit("count", _work).to_response()
63
+
64
+
65
+ @router.post("/api/jobs/index")
66
+ def start_index_job(payload: IndexJobRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
67
+ """Submit incremental sparse-index construction for a line dataset."""
68
+ path = resolve_data_file(payload.file)
69
+
70
+ def _work(context: JobContext) -> dict[str, Any]:
71
+ return {"file": payload.file, **build_line_index_with_progress(path, context)}
72
+
73
+ return _job_store(request).submit("index", _work).to_response()
74
+
75
+
76
+ @router.post("/api/jobs/search")
77
+ def start_search_job(payload: SearchJobRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
78
+ """Submit a cancellable bounded search with fingerprinted cache reuse."""
79
+ path = resolve_data_file(payload.file)
80
+ deleted_ids = deleted_row_ids_for(path)
81
+ search_term = payload.query.strip()
82
+ limit_value, _ = normalize_pagination(payload.limit, 0)
83
+
84
+ def _work(context: JobContext) -> dict[str, Any]:
85
+ cache_path = search_cache_path(path, query=search_term, limit=limit_value, deleted_ids=deleted_ids)
86
+ cached = load_cached_result(cache_path)
87
+ if cached is not None and cached.get("query") == search_term:
88
+ context.update(progress=1.0, message="Search loaded from cache")
89
+ return {**cached, "cached": True}
90
+ result = search_dataset(payload.file, path, query=search_term, limit=limit_value, control=context, deleted_ids=deleted_ids)
91
+ write_cached_result(cache_path, result)
92
+ return {**result, "cached": False}
93
+
94
+ return _job_store(request).submit("search", _work).to_response()
95
+
96
+
97
+ @router.post("/api/jobs/query")
98
+ def start_query_job(payload: QueryRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
99
+ """Submit validated read-only SQL under query resource limits."""
100
+ path = resolve_data_file(payload.file)
101
+
102
+ def _work(context: JobContext) -> dict[str, Any]:
103
+ return execute_query_guarded(
104
+ file_name=payload.file,
105
+ path=path,
106
+ sql=payload.sql,
107
+ limit=payload.limit,
108
+ offset=payload.offset,
109
+ context=context,
110
+ )
111
+
112
+ return _job_store(request).submit("query", _work).to_response()
113
+
114
+
115
+ @router.post("/api/jobs/stats")
116
+ def start_stats_job(payload: StatsJobRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
117
+ """Submit bounded column sampling with optional cache refresh."""
118
+ path = resolve_data_file(payload.file)
119
+
120
+ def _work(context: JobContext) -> dict[str, Any]:
121
+ context.update(progress=0.15, message="Sampling rows")
122
+ result = compute_cached_column_stats(payload.file, path, payload.sample, bool(payload.force))
123
+ context.update(progress=1.0, message="Stats ready")
124
+ return result
125
+
126
+ return _job_store(request).submit("stats", _work).to_response()
127
+
128
+
129
+ @router.post("/api/jobs/eda")
130
+ def start_eda_job(payload: EdaRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
131
+ """Submit EDA generation for a bounded dataset sample."""
132
+ path = resolve_data_file(payload.file)
133
+ reports = eda_reports_service()
134
+
135
+ def _work(context: JobContext) -> dict[str, Any]:
136
+ return reports.generate_dataset_eda_report(
137
+ file_name=payload.file,
138
+ path=path,
139
+ sample=payload.sample,
140
+ mode=payload.mode,
141
+ force=payload.force,
142
+ context=context,
143
+ )
144
+
145
+ return _job_store(request).submit("eda", _work).to_response()
146
+
147
+
148
+ @router.post("/api/jobs/eda_query")
149
+ def start_eda_query_job(payload: EdaQueryRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
150
+ """Submit EDA generation for validated SQL query results."""
151
+ path = resolve_data_file(payload.file)
152
+ reports = eda_reports_service()
153
+
154
+ def _work(context: JobContext) -> dict[str, Any]:
155
+ return reports.generate_query_eda_report(
156
+ file_name=payload.file,
157
+ path=path,
158
+ sql=payload.sql,
159
+ sample=payload.sample,
160
+ mode=payload.mode,
161
+ force=payload.force,
162
+ context=context,
163
+ )
164
+
165
+ return _job_store(request).submit("eda_query", _work).to_response()
166
+
167
+
168
+ @router.post("/api/jobs/atlas")
169
+ def start_atlas_job(payload: AtlasRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
170
+ """Submit embedding and Atlas startup for a dataset column."""
171
+ path = resolve_data_file(payload.file)
172
+ atlas = atlas_service()
173
+
174
+ def _work(context: JobContext) -> dict[str, Any]:
175
+ return atlas.run_atlas_visualization(
176
+ file_name=payload.file,
177
+ path=path,
178
+ column=payload.column,
179
+ model=payload.model,
180
+ backend=payload.backend,
181
+ prompt=payload.prompt,
182
+ projection_method=payload.projection_method,
183
+ sql=None,
184
+ sample=payload.sample,
185
+ context=context,
186
+ runtime=_atlas_runtime(request),
187
+ )
188
+
189
+ return _job_store(request).submit("atlas", _work).to_response()
190
+
191
+
192
+ @router.post("/api/jobs/atlas_query")
193
+ def start_atlas_query_job(payload: AtlasQueryRequest, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
194
+ """Submit embedding and Atlas startup for validated SQL results."""
195
+ path = resolve_data_file(payload.file)
196
+ atlas = atlas_service()
197
+
198
+ def _work(context: JobContext) -> dict[str, Any]:
199
+ return atlas.run_atlas_visualization(
200
+ file_name=payload.file,
201
+ path=path,
202
+ column=payload.column,
203
+ model=payload.model,
204
+ backend=payload.backend,
205
+ prompt=payload.prompt,
206
+ projection_method=payload.projection_method,
207
+ sql=payload.sql,
208
+ sample=payload.sample,
209
+ context=context,
210
+ runtime=_atlas_runtime(request),
211
+ )
212
+
213
+ return _job_store(request).submit("atlas_query", _work).to_response()
214
+
215
+
216
+ @router.get("/api/jobs/{job_id}")
217
+ def get_job(job_id: str, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
218
+ """Return a detached snapshot of a job, or HTTP 404 when unknown."""
219
+ record = _job_store(request).get(job_id)
220
+ if record is None:
221
+ raise HTTPException(status_code=404, detail="job not found")
222
+ return record.to_response()
223
+
224
+
225
+ @router.post("/api/jobs/{job_id}/cancel")
226
+ def cancel_job(job_id: str, request: Request = REQUEST_DEFAULT) -> dict[str, Any]:
227
+ """Request cooperative cancellation, or return HTTP 404 when unknown."""
228
+ record = _job_store(request).cancel(job_id)
229
+ if record is None:
230
+ raise HTTPException(status_code=404, detail="job not found")
231
+ return record.to_response()
@@ -0,0 +1,63 @@
1
+ """Dataset row and column mutation endpoints."""
2
+
3
+ from pathlib import Path
4
+ from typing import Any
5
+
6
+ from fastapi import APIRouter, HTTPException
7
+
8
+ from ..config import ALLOW_DELETE_DATA, HARD_DELETE_MAX_BYTES
9
+ from ..db import count_relation_rows_raw
10
+ from ..delete_ops import delete_column_from_file, delete_row_from_file
11
+ from ..deleted_rows import add_deleted_row_id, clear_deleted_row_ids, deleted_row_ids_for
12
+ from ..files import resolve_data_file
13
+ from .schemas import DeleteColumnRequest, DeleteRowRequest
14
+
15
+ router = APIRouter()
16
+
17
+
18
+ def _reject_large_hard_delete(path: Path) -> None:
19
+ if path.stat().st_size > HARD_DELETE_MAX_BYTES:
20
+ raise HTTPException(
21
+ status_code=400,
22
+ detail="delete from file rewrites the dataset and is disabled for large files; use hide only",
23
+ )
24
+
25
+
26
+ @router.post("/api/delete_row")
27
+ def delete_row(payload: DeleteRowRequest) -> dict[str, Any]:
28
+ """Hide a row or rewrite a small dataset when persistent deletion is allowed.
29
+
30
+ Row IDs are one-based. Persistent deletion clears all soft-deleted row IDs.
31
+ """
32
+ path = resolve_data_file(payload.file)
33
+ row_id = int(payload.row_id)
34
+ if row_id < 1:
35
+ raise HTTPException(status_code=400, detail="row_id must be positive")
36
+ if payload.persist:
37
+ if not ALLOW_DELETE_DATA:
38
+ raise HTTPException(status_code=403, detail="delete from file is disabled")
39
+ _reject_large_hard_delete(path)
40
+ total_rows = count_relation_rows_raw(path)
41
+ if row_id > total_rows:
42
+ raise HTTPException(status_code=404, detail="row not found")
43
+ delete_row_from_file(path, row_id)
44
+ clear_deleted_row_ids(path)
45
+ return {"file": payload.file, "row_id": row_id, "persisted": True, "total_rows": max(0, total_rows - 1)}
46
+ add_deleted_row_id(path, row_id)
47
+ return {"file": payload.file, "row_id": row_id, "persisted": False, "deleted_count": len(deleted_row_ids_for(path))}
48
+
49
+
50
+ @router.post("/api/delete_column")
51
+ def delete_column(payload: DeleteColumnRequest) -> dict[str, Any]:
52
+ """Acknowledge a soft column hide or persistently rewrite a small dataset."""
53
+ path = resolve_data_file(payload.file)
54
+ column = payload.column.strip()
55
+ if not column:
56
+ raise HTTPException(status_code=400, detail="column is required")
57
+ if payload.persist:
58
+ if not ALLOW_DELETE_DATA:
59
+ raise HTTPException(status_code=403, detail="delete from file is disabled")
60
+ _reject_large_hard_delete(path)
61
+ delete_column_from_file(path, column)
62
+ return {"file": payload.file, "column": column, "persisted": True}
63
+ return {"file": payload.file, "column": column, "persisted": False}
@@ -0,0 +1,117 @@
1
+ """Request schemas shared by Local Data Studio API routers."""
2
+
3
+ from typing import Any, Literal
4
+
5
+ from pydantic import BaseModel, Field, field_validator
6
+
7
+
8
+ class QueryRequest(BaseModel):
9
+ """Input for a guarded read-only SQL query."""
10
+
11
+ file: str = Field(min_length=1)
12
+ sql: str = Field(min_length=1)
13
+ limit: int | None = Field(default=None, ge=1)
14
+ offset: int | None = Field(default=None, ge=0)
15
+
16
+
17
+ class RawRowRequest(BaseModel):
18
+ """Input selecting either a one-based dataset row or SQL result offset."""
19
+
20
+ file: str = Field(min_length=1)
21
+ row_id: int | None = Field(default=None, ge=1)
22
+ sql: str | None = None
23
+ offset: int | None = Field(default=None, ge=0)
24
+
25
+
26
+ class EdaRequest(BaseModel):
27
+ """Input controlling bounded EDA generation and report cache reuse."""
28
+
29
+ file: str = Field(min_length=1)
30
+ sample: int | None = None
31
+ force: bool | None = None
32
+ mode: Literal["minimal", "maximal"] | None = None
33
+
34
+ @field_validator("sample")
35
+ @classmethod
36
+ def validate_sample(cls, value: int | None) -> int | None:
37
+ """Accept the unlimited marker or a positive row limit."""
38
+ if value is None or value == -1 or value >= 1:
39
+ return value
40
+ raise ValueError("sample must be -1 or an integer greater than or equal to 1")
41
+
42
+
43
+ class EdaQueryRequest(EdaRequest):
44
+ """EDA input extended with the guarded SQL that supplies report rows."""
45
+
46
+ sql: str = Field(min_length=1)
47
+
48
+
49
+ class AtlasRequest(BaseModel):
50
+ """Input selecting a dataset column and local model for Atlas embedding."""
51
+
52
+ file: str = Field(min_length=1)
53
+ column: str = Field(min_length=1)
54
+ model: str = Field(min_length=1)
55
+ sample: int | None = Field(default=None, ge=0)
56
+ backend: Literal["transformers", "sentence-transformers"] | None = None
57
+ prompt: str | None = Field(default=None, max_length=16_384)
58
+ projection_method: Literal["umap", "tsne", "pca"] = "umap"
59
+
60
+
61
+ class AtlasQueryRequest(AtlasRequest):
62
+ """Atlas input extended with the guarded SQL that supplies embedding rows."""
63
+
64
+ sql: str = Field(min_length=1)
65
+
66
+
67
+ class CountJobRequest(BaseModel):
68
+ """Input for a background row-count operation."""
69
+
70
+ file: str = Field(min_length=1)
71
+
72
+
73
+ class IndexJobRequest(BaseModel):
74
+ """Input for incremental sparse-index construction."""
75
+
76
+ file: str = Field(min_length=1)
77
+
78
+
79
+ class SearchJobRequest(BaseModel):
80
+ """Input for a bounded background dataset search."""
81
+
82
+ file: str = Field(min_length=1)
83
+ query: str = Field(min_length=1)
84
+ limit: int | None = Field(default=None, ge=1)
85
+
86
+
87
+ class StatsJobRequest(BaseModel):
88
+ """Input controlling sampled statistics and cache refresh."""
89
+
90
+ file: str = Field(min_length=1)
91
+ sample: int | None = None
92
+ force: bool | None = None
93
+
94
+
95
+ class DeleteRowRequest(BaseModel):
96
+ """Input for one-based soft or persistent row deletion."""
97
+
98
+ file: str = Field(min_length=1)
99
+ row_id: int = Field(ge=1)
100
+ persist: bool | None = None
101
+
102
+
103
+ class DeleteColumnRequest(BaseModel):
104
+ """Input for soft or persistent column deletion."""
105
+
106
+ file: str = Field(min_length=1)
107
+ column: str = Field(min_length=1)
108
+ persist: bool | None = None
109
+
110
+
111
+ class NLQueryRequest(BaseModel):
112
+ """Input for natural-language SQL generation with optional sample context."""
113
+
114
+ file: str = Field(min_length=1)
115
+ prompt: str = Field(min_length=1, max_length=16_384)
116
+ sample: dict[str, Any] | None = None
117
+ model: str | None = Field(default=None, max_length=64)
@@ -0,0 +1,99 @@
1
+ """Small shared services used by API routers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import tempfile
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from fastapi import HTTPException
12
+
13
+ from ..cache import stats_cache_path
14
+ from ..config import DEFAULT_SAMPLE
15
+ from ..sql import is_large_dataset
16
+ from ..stats import compute_column_stats
17
+
18
+
19
+ def atlas_service() -> Any:
20
+ """Import the optional ML stack only when an Atlas job is requested."""
21
+ from .. import atlas # noqa: PLC0415
22
+
23
+ return atlas
24
+
25
+
26
+ def eda_reports_service() -> Any:
27
+ """Import profiling dependencies only when an EDA operation is requested."""
28
+ try:
29
+ from .. import eda_reports # noqa: PLC0415
30
+ except ImportError as exc:
31
+ raise HTTPException(status_code=500, detail=f"EDA dependencies could not be loaded: {exc}") from exc
32
+ return eda_reports
33
+
34
+
35
+ def reject_large_sync_operation(path: Path, operation: str) -> None:
36
+ """Reject potentially blocking full scans above the large-dataset threshold."""
37
+ if is_large_dataset(path):
38
+ raise HTTPException(
39
+ status_code=400,
40
+ detail=f"{operation} can scan the full dataset; use the background job endpoint for large files",
41
+ )
42
+
43
+
44
+ def load_cached_result(cache_path: Path) -> dict[str, Any] | None:
45
+ """Load an object-shaped JSON cache, treating missing or invalid data as a miss."""
46
+ if not cache_path.exists():
47
+ return None
48
+ try:
49
+ payload = json.loads(cache_path.read_text(encoding="utf-8"))
50
+ except (OSError, json.JSONDecodeError):
51
+ return None
52
+ return payload if isinstance(payload, dict) else None
53
+
54
+
55
+ def write_cached_result(cache_path: Path, result: dict[str, Any]) -> None:
56
+ """Atomically replace a JSON cache file with the supplied result.
57
+
58
+ The caller retains ownership of ``result``; this function does not mutate it.
59
+ """
60
+ cache_path.parent.mkdir(parents=True, exist_ok=True)
61
+ temporary_path: Path | None = None
62
+ try:
63
+ with tempfile.NamedTemporaryFile(
64
+ mode="w",
65
+ encoding="utf-8",
66
+ dir=cache_path.parent,
67
+ prefix=f".{cache_path.name}.",
68
+ suffix=".tmp",
69
+ delete=False,
70
+ ) as temporary:
71
+ json.dump(result, temporary, ensure_ascii=False, indent=2)
72
+ temporary.flush()
73
+ os.fsync(temporary.fileno())
74
+ temporary_path = Path(temporary.name)
75
+ os.replace(temporary_path, cache_path)
76
+ temporary_path = None
77
+ finally:
78
+ if temporary_path is not None:
79
+ temporary_path.unlink(missing_ok=True)
80
+
81
+
82
+ def _load_cached_stats(path: Path, sample: int) -> dict[str, Any] | None:
83
+ payload = load_cached_result(stats_cache_path(path))
84
+ if payload is None or payload.get("sample_request") != sample:
85
+ return None
86
+ result = payload.get("result")
87
+ return result if isinstance(result, dict) else None
88
+
89
+
90
+ def compute_cached_column_stats(file_name: str, path: Path, sample: int | None, force: bool) -> dict[str, Any]:
91
+ """Compute or reuse statistics keyed by dataset fingerprint and sample size."""
92
+ sample_value = sample if sample is not None else DEFAULT_SAMPLE
93
+ if not force:
94
+ cached = _load_cached_stats(path, sample_value)
95
+ if cached is not None:
96
+ return {**cached, "cached": True}
97
+ result = compute_column_stats(file_name, path, sample_value)
98
+ write_cached_result(stats_cache_path(path), {"sample_request": sample_value, "result": result})
99
+ return {**result, "cached": False}