sqlseed-web 0.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlseed_web/AGENTS.md +106 -0
- sqlseed_web/__init__.py +24 -0
- sqlseed_web/__main__.py +8 -0
- sqlseed_web/_application.py +205 -0
- sqlseed_web/ai_settings.py +290 -0
- sqlseed_web/api.py +1102 -0
- sqlseed_web/app.py +26 -0
- sqlseed_web/managed_worker.py +184 -0
- sqlseed_web/operation_errors.py +42 -0
- sqlseed_web/plugin_environment.py +231 -0
- sqlseed_web/plugin_management.py +322 -0
- sqlseed_web/plugin_process.py +131 -0
- sqlseed_web/runtime_lifecycle.py +130 -0
- sqlseed_web/runtime_session.py +97 -0
- sqlseed_web/settings_environment.py +368 -0
- sqlseed_web/sqlite_target.py +86 -0
- sqlseed_web/state.py +348 -0
- sqlseed_web/static/AGENTS.md +180 -0
- sqlseed_web/static/ai.css +57 -0
- sqlseed_web/static/configs.css +50 -0
- sqlseed_web/static/date-picker.css +230 -0
- sqlseed_web/static/disclosure.css +149 -0
- sqlseed_web/static/graph-clarity.css +103 -0
- sqlseed_web/static/index.html +29 -0
- sqlseed_web/static/js/api.js +228 -0
- sqlseed_web/static/js/app.js +97 -0
- sqlseed_web/static/js/dropdown.js +432 -0
- sqlseed_web/static/js/filepicker.js +203 -0
- sqlseed_web/static/js/genform.js +1066 -0
- sqlseed_web/static/js/labels.js +195 -0
- sqlseed_web/static/js/pages/browse.js +209 -0
- sqlseed_web/static/js/pages/configs.js +424 -0
- sqlseed_web/static/js/pages/connect.js +332 -0
- sqlseed_web/static/js/pages/heal.js +395 -0
- sqlseed_web/static/js/pages/meta.js +110 -0
- sqlseed_web/static/js/pages/runs.js +293 -0
- sqlseed_web/static/js/pages/settings.js +942 -0
- sqlseed_web/static/js/pages/wizard.js +751 -0
- sqlseed_web/static/js/pages/workbench.js +3123 -0
- sqlseed_web/static/js/tree.js +126 -0
- sqlseed_web/static/js/workbench/ai-eligibility.js +33 -0
- sqlseed_web/static/js/workbench/ai-handoff.js +31 -0
- sqlseed_web/static/js/workbench/ai-stream.js +116 -0
- sqlseed_web/static/js/workbench/ai.js +888 -0
- sqlseed_web/static/js/workbench/connection.js +508 -0
- sqlseed_web/static/js/workbench/date-picker.js +445 -0
- sqlseed_web/static/js/workbench/dependency-view.js +119 -0
- sqlseed_web/static/js/workbench/editor.js +1236 -0
- sqlseed_web/static/js/workbench/focus.js +11 -0
- sqlseed_web/static/js/workbench/graph-layout.js +332 -0
- sqlseed_web/static/js/workbench/graph.js +970 -0
- sqlseed_web/static/js/workbench/guidance.js +29 -0
- sqlseed_web/static/js/workbench/model.js +124 -0
- sqlseed_web/static/js/workbench/plugin-management.js +512 -0
- sqlseed_web/static/js/workbench/preview-scroll-layout.js +94 -0
- sqlseed_web/static/js/workbench/preview.js +572 -0
- sqlseed_web/static/js/workbench/provider-guide.js +33 -0
- sqlseed_web/static/js/workbench/recovery.js +28 -0
- sqlseed_web/static/js/workbench/scroll-lock.js +26 -0
- sqlseed_web/static/js/workbench/session.js +174 -0
- sqlseed_web/static/js/workbench/table-data.js +186 -0
- sqlseed_web/static/js/workbench/ui.js +262 -0
- sqlseed_web/static/navigation.css +92 -0
- sqlseed_web/static/preview.css +29 -0
- sqlseed_web/static/runs.css +53 -0
- sqlseed_web/static/scrollbars.css +42 -0
- sqlseed_web/static/settings.css +108 -0
- sqlseed_web/static/style.css +3382 -0
- sqlseed_web/static/table-data.css +27 -0
- sqlseed_web/static/workbench.css +509 -0
- sqlseed_web/supervised_plugins.py +173 -0
- sqlseed_web/supervisor.py +238 -0
- sqlseed_web/workbench.py +381 -0
- sqlseed_web/workbench_ai.py +887 -0
- sqlseed_web/workbench_ai_relations.py +285 -0
- sqlseed_web/workbench_ai_stream.py +172 -0
- sqlseed_web/workbench_data.py +163 -0
- sqlseed_web/workbench_execution.py +199 -0
- sqlseed_web/workbench_runtime.py +1218 -0
- sqlseed_web/workbench_schema.py +277 -0
- sqlseed_web/workbench_store.py +458 -0
- sqlseed_web/worker_control.py +192 -0
- sqlseed_web-0.2.4.dist-info/METADATA +105 -0
- sqlseed_web-0.2.4.dist-info/RECORD +87 -0
- sqlseed_web-0.2.4.dist-info/WHEEL +4 -0
- sqlseed_web-0.2.4.dist-info/entry_points.txt +2 -0
- sqlseed_web-0.2.4.dist-info/licenses/LICENSE +679 -0
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
"""Finite AI relation templates compiled by the server, never by the model."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from copy import deepcopy
|
|
8
|
+
from typing import Any, Literal
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
11
|
+
from sqlglot import exp, parse_one
|
|
12
|
+
from sqlglot.errors import SqlglotError
|
|
13
|
+
from sqlseed._utils.type_checks import has_exact_type
|
|
14
|
+
from sqlseed.config.models import ColumnConfig
|
|
15
|
+
from sqlseed.core.check_parser import CheckConstraintParser, ParsedCheck
|
|
16
|
+
from sqlseed.core.column_dag import ColumnDAG
|
|
17
|
+
from sqlseed.core.mapper import GeneratorSpec
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class RelationSuggestion(BaseModel):
|
|
21
|
+
model_config = ConfigDict(extra="forbid")
|
|
22
|
+
kind: Literal["relation"]
|
|
23
|
+
table: str = Field(min_length=1, max_length=300)
|
|
24
|
+
column: str = Field(min_length=1, max_length=300)
|
|
25
|
+
template: Literal["copy", "concat", "product", "date_offset"]
|
|
26
|
+
sources: list[str] = Field(min_length=1, max_length=8)
|
|
27
|
+
options: dict[str, Any] = Field(default_factory=dict)
|
|
28
|
+
reason: str = Field(default="请确认同一行字段之间的业务关系。", max_length=2000)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def locked_column(
|
|
32
|
+
table: dict[str, Any], column: dict[str, Any], rule: dict[str, Any] | None, *, source: bool = False
|
|
33
|
+
) -> bool:
|
|
34
|
+
"""Protect managed rules; skipped values cannot supply a relation source."""
|
|
35
|
+
foreign = {name for key in table["foreign_keys"] for name in key["columns"]}
|
|
36
|
+
protected: tuple[str, ...] = ("faker_method", "mimesis_method", "native_params")
|
|
37
|
+
if not source:
|
|
38
|
+
protected += ("derive_from", "expression")
|
|
39
|
+
generator = (rule or {}).get("generator") or table.get("mapping", {}).get(column["name"], {}).get("generator_name")
|
|
40
|
+
# An existing derived rule supplies a value even if the SQL column also
|
|
41
|
+
# declares DEFAULT. It may feed another relation but remains a locked target.
|
|
42
|
+
derived = bool(rule and rule.get("derive_from") and rule.get("expression"))
|
|
43
|
+
uses_default = (
|
|
44
|
+
not derived
|
|
45
|
+
and column.get("default") is not None
|
|
46
|
+
and (not generator or generator == "skip" or str(generator).startswith("__"))
|
|
47
|
+
)
|
|
48
|
+
return bool(
|
|
49
|
+
column["name"] in foreign
|
|
50
|
+
or column.get("is_primary_key")
|
|
51
|
+
or column.get("is_computed")
|
|
52
|
+
or uses_default
|
|
53
|
+
or (rule and (any(rule.get(key) for key in protected) or (source and rule.get("generator") == "skip")))
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _kind(column: dict[str, Any]) -> str:
|
|
58
|
+
name = str(column["type"]).upper()
|
|
59
|
+
if "INT" in name:
|
|
60
|
+
return "integer"
|
|
61
|
+
if any(token in name for token in ("REAL", "FLOAT", "DOUBLE", "NUMERIC", "DECIMAL")):
|
|
62
|
+
return "number"
|
|
63
|
+
if "DATETIME" in name or "TIMESTAMP" in name:
|
|
64
|
+
return "datetime"
|
|
65
|
+
if name == "DATE":
|
|
66
|
+
return "date"
|
|
67
|
+
if any(token in name for token in ("TEXT", "CHAR", "STRING")):
|
|
68
|
+
return "text"
|
|
69
|
+
return name
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _copy_expression(kinds: list[str], target_kind: str, options: dict[str, Any]) -> str:
|
|
73
|
+
if (
|
|
74
|
+
options
|
|
75
|
+
or len(kinds) != 1
|
|
76
|
+
or not (kinds[0] == target_kind or (kinds[0] == "integer" and target_kind == "number"))
|
|
77
|
+
):
|
|
78
|
+
raise ValueError("复制需要兼容的单个来源")
|
|
79
|
+
return "value"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _concat_expression(kinds: list[str], target_kind: str, options: dict[str, Any], values: list[str]) -> str:
|
|
83
|
+
separator = options.get("separator", "")
|
|
84
|
+
if (
|
|
85
|
+
set(options) - {"separator"}
|
|
86
|
+
or not isinstance(separator, str)
|
|
87
|
+
or len(separator) > 32
|
|
88
|
+
or target_kind != "text"
|
|
89
|
+
or any(kind != "text" for kind in kinds)
|
|
90
|
+
):
|
|
91
|
+
raise ValueError("拼接仅支持文本字段与不超过 32 字的分隔符")
|
|
92
|
+
return (" + " + json.dumps(separator, ensure_ascii=False) + " + ").join(values)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _product_expression(kinds: list[str], target_kind: str, options: dict[str, Any], values: list[str]) -> str:
|
|
96
|
+
precision = options.get("precision", 2 if target_kind == "number" else 0)
|
|
97
|
+
if set(options) - {"precision"} or not has_exact_type(precision, int) or not 0 <= precision <= 8:
|
|
98
|
+
raise ValueError("乘积需要两个数值字段,精度为 0–8")
|
|
99
|
+
if (
|
|
100
|
+
len(kinds) != 2
|
|
101
|
+
or any(kind not in {"integer", "number"} for kind in kinds)
|
|
102
|
+
or target_kind not in {"integer", "number"}
|
|
103
|
+
):
|
|
104
|
+
raise ValueError("乘积需要两个数值字段,精度为 0–8")
|
|
105
|
+
if target_kind == "integer" and (precision != 0 or any(kind != "integer" for kind in kinds)):
|
|
106
|
+
raise ValueError("整数目标需要整数来源")
|
|
107
|
+
return f"round({values[0]} * {values[1]}, {precision})"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _date_offset_expression(kinds: list[str], target_kind: str, options: dict[str, Any]) -> str:
|
|
111
|
+
days = options.get("days", 0)
|
|
112
|
+
if set(options) - {"days"} or not has_exact_type(days, int) or not -36500 <= days <= 36500:
|
|
113
|
+
raise ValueError("日期偏移需要相同日期类型,天数为 -36500–36500")
|
|
114
|
+
if len(kinds) != 1 or target_kind not in {"date", "datetime"} or kinds[0] != target_kind:
|
|
115
|
+
raise ValueError("日期偏移需要相同日期类型,天数为 -36500–36500")
|
|
116
|
+
return f"value + timedelta(days={days})"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _relation_expression(suggestion: RelationSuggestion, kinds: list[str], target_kind: str, values: list[str]) -> str:
|
|
120
|
+
if suggestion.template == "copy":
|
|
121
|
+
return _copy_expression(kinds, target_kind, suggestion.options)
|
|
122
|
+
if suggestion.template == "concat":
|
|
123
|
+
return _concat_expression(kinds, target_kind, suggestion.options, values)
|
|
124
|
+
if suggestion.template == "product":
|
|
125
|
+
return _product_expression(kinds, target_kind, suggestion.options, values)
|
|
126
|
+
return _date_offset_expression(kinds, target_kind, suggestion.options)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def compile_relation(
|
|
130
|
+
suggestion: RelationSuggestion, table: dict[str, Any], rules: dict[str, dict[str, Any]]
|
|
131
|
+
) -> dict[str, Any]:
|
|
132
|
+
"""Compile only bounded typed templates, with explicit NULL propagation."""
|
|
133
|
+
columns = {col["name"]: col for col in table["columns"]}
|
|
134
|
+
target = columns[suggestion.column]
|
|
135
|
+
sources = [columns[name] for name in suggestion.sources]
|
|
136
|
+
if len(set(suggestion.sources)) != len(sources) or suggestion.column in suggestion.sources:
|
|
137
|
+
raise ValueError("来源不能重复或引用自身")
|
|
138
|
+
if any(locked_column(table, col, rules.get(col["name"]), source=True) for col in sources):
|
|
139
|
+
raise ValueError("来源必须是同一行中可生成的普通或派生字段")
|
|
140
|
+
if not target.get("nullable", True) and any(
|
|
141
|
+
col.get("nullable", True) or (rules.get(col["name"], {}).get("null_ratio") or 0) > 0 for col in sources
|
|
142
|
+
):
|
|
143
|
+
raise ValueError("可空来源不能派生为不可空目标")
|
|
144
|
+
kinds, target_kind = [_kind(col) for col in sources], _kind(target)
|
|
145
|
+
values = ["value"] if len(sources) == 1 else [f"value[{index}]" for index in range(len(sources))]
|
|
146
|
+
expression = _relation_expression(suggestion, kinds, target_kind, values)
|
|
147
|
+
if suggestion.template != "copy":
|
|
148
|
+
expression = f"None if {' or '.join(value + ' == None' for value in values)} else ({expression})"
|
|
149
|
+
after = deepcopy(rules.get(suggestion.column, {}))
|
|
150
|
+
for key in ("generator", "params", "provider", "faker_method", "mimesis_method", "native_params"):
|
|
151
|
+
after.pop(key, None)
|
|
152
|
+
after.update(
|
|
153
|
+
name=suggestion.column,
|
|
154
|
+
derive_from=suggestion.sources[0] if len(sources) == 1 else suggestion.sources,
|
|
155
|
+
expression=expression,
|
|
156
|
+
)
|
|
157
|
+
ColumnConfig.model_validate(after)
|
|
158
|
+
return after
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def validate_dags(document: dict[str, Any], schema: dict[str, Any]) -> None:
|
|
162
|
+
"""Use the real core DAG, retaining all old derived rules and row references."""
|
|
163
|
+
tables = {table["name"]: table for table in schema["tables"]}
|
|
164
|
+
for table in document["tables"]:
|
|
165
|
+
columns = {col["name"] for col in tables[table["name"]]["columns"]}
|
|
166
|
+
configs = [ColumnConfig.model_validate(col) for col in table.get("columns", [])]
|
|
167
|
+
for config in configs:
|
|
168
|
+
if isinstance(config.derive_from, list):
|
|
169
|
+
sources = config.derive_from
|
|
170
|
+
elif config.derive_from:
|
|
171
|
+
sources = [config.derive_from]
|
|
172
|
+
else:
|
|
173
|
+
sources = []
|
|
174
|
+
if not set(sources) <= columns:
|
|
175
|
+
raise ValueError("派生来源不存在")
|
|
176
|
+
ColumnDAG().build({name: GeneratorSpec(generator_name="string") for name in columns}, configs)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _constraint_columns(table: dict[str, Any]) -> list[list[str]]:
|
|
180
|
+
constraints = [constraint["columns"] for constraint in table["unique_constraints"]]
|
|
181
|
+
for check in table["checks"]:
|
|
182
|
+
try:
|
|
183
|
+
columns = [column.name for column in parse_one(check["expression"]).find_all(exp.Column)]
|
|
184
|
+
except SqlglotError:
|
|
185
|
+
# If a dialect-specific constraint cannot be understood, keep
|
|
186
|
+
# the table's proposals together rather than guessing independence.
|
|
187
|
+
columns = [column["name"] for column in table["columns"]]
|
|
188
|
+
constraints.append(columns)
|
|
189
|
+
return constraints
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _patch_dependencies(
|
|
193
|
+
document: dict[str, Any], schema: dict[str, Any]
|
|
194
|
+
) -> Iterator[tuple[tuple[str, str], tuple[str, str]]]:
|
|
195
|
+
for table in document["tables"]:
|
|
196
|
+
configs = [ColumnConfig.model_validate(col) for col in table.get("columns", [])]
|
|
197
|
+
specs = {col.name: GeneratorSpec(generator_name="string") for col in configs}
|
|
198
|
+
for node in ColumnDAG().build(specs, configs):
|
|
199
|
+
for source in node.depends_on:
|
|
200
|
+
yield (table["name"], source), (table["name"], node.name)
|
|
201
|
+
for table in schema["tables"]:
|
|
202
|
+
for columns in _constraint_columns(table):
|
|
203
|
+
for name in columns[1:]:
|
|
204
|
+
yield (table["name"], name), (table["name"], columns[0])
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def group_patches(patches: list[dict[str, Any]], document: dict[str, Any], schema: dict[str, Any]) -> None:
|
|
208
|
+
"""Connected old/new derivations are a single review and application unit."""
|
|
209
|
+
parents: dict[tuple[str, str], tuple[str, str]] = {}
|
|
210
|
+
|
|
211
|
+
def root(node: tuple[str, str]) -> tuple[str, str]:
|
|
212
|
+
parents.setdefault(node, node)
|
|
213
|
+
if parents[node] != node:
|
|
214
|
+
parents[node] = root(parents[node])
|
|
215
|
+
return parents[node]
|
|
216
|
+
|
|
217
|
+
for source, target in _patch_dependencies(document, schema):
|
|
218
|
+
parents[root(source)] = root(target)
|
|
219
|
+
groups: dict[tuple[str, str], str] = {}
|
|
220
|
+
for patch in patches:
|
|
221
|
+
group_root = root((patch["table"], patch["column"]))
|
|
222
|
+
patch["group_id"] = groups.setdefault(group_root, f"ai-group-{len(groups) + 1}")
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
class SampleCheckError(ValueError):
|
|
226
|
+
"""Expose the failed schema constraint, never the generated value."""
|
|
227
|
+
|
|
228
|
+
def __init__(self, table: str, column: str, message: str) -> None:
|
|
229
|
+
super().__init__(f"{table}.{column}: {message}")
|
|
230
|
+
self.issue = {
|
|
231
|
+
"code": "sample_check_failed",
|
|
232
|
+
"severity": "error",
|
|
233
|
+
"table": table,
|
|
234
|
+
"column": column,
|
|
235
|
+
"message": message,
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _validate_sample_range(table: str, column: str, parsed: ParsedCheck, value: Any) -> None:
|
|
240
|
+
if parsed.min_value is not None and (
|
|
241
|
+
value < parsed.min_value or (parsed.min_exclusive and value == parsed.min_value)
|
|
242
|
+
):
|
|
243
|
+
raise SampleCheckError(
|
|
244
|
+
table,
|
|
245
|
+
column,
|
|
246
|
+
f"生成值不满足 CHECK 下界({'>' if parsed.min_exclusive else '>='} {parsed.min_value})",
|
|
247
|
+
)
|
|
248
|
+
if parsed.max_value is not None and (
|
|
249
|
+
value > parsed.max_value or (parsed.max_exclusive and value == parsed.max_value)
|
|
250
|
+
):
|
|
251
|
+
raise SampleCheckError(
|
|
252
|
+
table,
|
|
253
|
+
column,
|
|
254
|
+
f"生成值不满足 CHECK 上界({'<' if parsed.max_exclusive else '<='} {parsed.max_value})",
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _validate_sample_value(table: str, column: str, parsed: ParsedCheck, value: Any) -> None:
|
|
259
|
+
if value is None:
|
|
260
|
+
return # SQL CHECK accepts UNKNOWN; NOT NULL is checked separately.
|
|
261
|
+
if parsed.kind == "choice" and value not in parsed.choices:
|
|
262
|
+
raise SampleCheckError(table, column, "生成值不满足 CHECK 候选范围")
|
|
263
|
+
if parsed.kind == "range":
|
|
264
|
+
_validate_sample_range(table, column, parsed, value)
|
|
265
|
+
if parsed.kind == "length_range" and (
|
|
266
|
+
(parsed.min_length is not None and len(value) < parsed.min_length)
|
|
267
|
+
or (parsed.max_length is not None and len(value) > parsed.max_length)
|
|
268
|
+
):
|
|
269
|
+
raise SampleCheckError(
|
|
270
|
+
table,
|
|
271
|
+
column,
|
|
272
|
+
f"生成值长度不满足 CHECK(最少 {parsed.min_length if parsed.min_length is not None else 0},"
|
|
273
|
+
f"最多 {parsed.max_length if parsed.max_length is not None else '不限'})",
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def validate_sample_checks(schema: dict[str, Any], samples: dict[str, Any]) -> None:
|
|
278
|
+
"""Check finite single-column constraints against actual generated values."""
|
|
279
|
+
for table in schema["tables"]:
|
|
280
|
+
for column in table["columns"]:
|
|
281
|
+
for check in table["checks"]:
|
|
282
|
+
if (parsed := CheckConstraintParser.parse(column["name"], check["expression"])) is None:
|
|
283
|
+
continue
|
|
284
|
+
for row in samples.get(table["name"], []):
|
|
285
|
+
_validate_sample_value(table["name"], column["name"], parsed, row.get(column["name"]))
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""Bounded, cancellable delivery of synchronous AI work without a database lease."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
from collections.abc import AsyncIterator, Callable
|
|
8
|
+
from concurrent.futures import Future
|
|
9
|
+
from queue import Empty, Full, Queue
|
|
10
|
+
from threading import Event, Lock
|
|
11
|
+
from time import monotonic
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from fastapi import HTTPException, Request
|
|
15
|
+
from fastapi.responses import Response, StreamingResponse
|
|
16
|
+
from sqlseed._utils.daemon_task import DaemonTask
|
|
17
|
+
|
|
18
|
+
from sqlseed_web import runtime_lifecycle
|
|
19
|
+
|
|
20
|
+
ANALYSIS_TIMEOUT = 180.0
|
|
21
|
+
_active: set[str] = set()
|
|
22
|
+
_active_lock = Lock()
|
|
23
|
+
Analysis = Callable[[Callable[[str, str], None], Callable[[], None]], dict[str, Any]]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _error(exc: HTTPException) -> dict[str, Any]:
|
|
27
|
+
detail = exc.detail
|
|
28
|
+
return {
|
|
29
|
+
"type": "error",
|
|
30
|
+
"code": detail.get("code", "ai_analysis_failed") if isinstance(detail, dict) else "ai_analysis_failed",
|
|
31
|
+
"message": detail.get("message", "AI 分析失败,请重试。") if isinstance(detail, dict) else str(detail),
|
|
32
|
+
"status": exc.status_code,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class AnalysisOperation:
|
|
37
|
+
"""Own the in-flight gate until the worker actually exits, even after disconnect."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, conn_id: str, run: Analysis) -> None:
|
|
40
|
+
self.cancelled = Event()
|
|
41
|
+
self.deadline = monotonic() + ANALYSIS_TIMEOUT
|
|
42
|
+
self.queue: Queue[dict[str, Any]] = Queue(maxsize=16)
|
|
43
|
+
with _active_lock:
|
|
44
|
+
if conn_id in _active:
|
|
45
|
+
raise HTTPException(
|
|
46
|
+
409, detail={"code": "ai_busy", "message": "此连接的 AI 分析尚未结束,请稍后重试。"}
|
|
47
|
+
)
|
|
48
|
+
_active.add(conn_id)
|
|
49
|
+
|
|
50
|
+
gate = runtime_lifecycle.runtime_gate
|
|
51
|
+
try:
|
|
52
|
+
gate.acquire("ai_analyses")
|
|
53
|
+
except BaseException:
|
|
54
|
+
with _active_lock:
|
|
55
|
+
_active.discard(conn_id)
|
|
56
|
+
raise
|
|
57
|
+
|
|
58
|
+
def worker() -> dict[str, Any]:
|
|
59
|
+
self.check_cancelled()
|
|
60
|
+
result = run(self.progress, self.check_cancelled)
|
|
61
|
+
self.check_cancelled()
|
|
62
|
+
return result
|
|
63
|
+
|
|
64
|
+
def finished(task: Future[dict[str, Any]]) -> None:
|
|
65
|
+
try:
|
|
66
|
+
if (failure := task.exception()) is None:
|
|
67
|
+
self.publish({"type": "result", "result": task.result()})
|
|
68
|
+
elif isinstance(failure, HTTPException):
|
|
69
|
+
self.publish(_error(failure))
|
|
70
|
+
else:
|
|
71
|
+
self.publish(
|
|
72
|
+
_error(
|
|
73
|
+
HTTPException(
|
|
74
|
+
500,
|
|
75
|
+
detail={
|
|
76
|
+
"code": "ai_analysis_failed",
|
|
77
|
+
"message": "AI 分析未完成,请重试;当前规则未改变。",
|
|
78
|
+
},
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
finally:
|
|
83
|
+
with _active_lock:
|
|
84
|
+
_active.discard(conn_id)
|
|
85
|
+
gate.release("ai_analyses")
|
|
86
|
+
|
|
87
|
+
try:
|
|
88
|
+
self.task = DaemonTask(worker, name="sqlseed-ai-analysis", on_done=finished)
|
|
89
|
+
except BaseException:
|
|
90
|
+
gate.release("ai_analyses")
|
|
91
|
+
with _active_lock:
|
|
92
|
+
_active.discard(conn_id)
|
|
93
|
+
raise
|
|
94
|
+
|
|
95
|
+
def publish(self, event: dict[str, Any]) -> None:
|
|
96
|
+
# There are four phase events and one terminal event. Keep the queue
|
|
97
|
+
# bounded even if a future caller reports more frequently.
|
|
98
|
+
try:
|
|
99
|
+
self.queue.put_nowait(event)
|
|
100
|
+
except Full:
|
|
101
|
+
try:
|
|
102
|
+
self.queue.get_nowait()
|
|
103
|
+
except Empty:
|
|
104
|
+
pass
|
|
105
|
+
self.queue.put_nowait(event)
|
|
106
|
+
|
|
107
|
+
def check_cancelled(self) -> None:
|
|
108
|
+
if self.cancelled.is_set():
|
|
109
|
+
raise HTTPException(499, detail={"code": "ai_cancelled", "message": "分析已取消。"})
|
|
110
|
+
if monotonic() >= self.deadline:
|
|
111
|
+
raise HTTPException(504, detail={"code": "ai_timeout", "message": "分析超过时间限制,请缩小范围后重试。"})
|
|
112
|
+
|
|
113
|
+
def progress(self, stage: str, message: str) -> None:
|
|
114
|
+
self.check_cancelled()
|
|
115
|
+
self.publish({"type": "progress", "stage": stage, "message": message})
|
|
116
|
+
|
|
117
|
+
async def events(self, request: Request | None = None) -> AsyncIterator[dict[str, Any]]:
|
|
118
|
+
try:
|
|
119
|
+
while True:
|
|
120
|
+
if request is not None and await request.is_disconnected():
|
|
121
|
+
return
|
|
122
|
+
if monotonic() >= self.deadline:
|
|
123
|
+
yield _error(
|
|
124
|
+
HTTPException(
|
|
125
|
+
504, detail={"code": "ai_timeout", "message": "分析超过时间限制,请缩小范围后重试。"}
|
|
126
|
+
)
|
|
127
|
+
)
|
|
128
|
+
return
|
|
129
|
+
try:
|
|
130
|
+
event = self.queue.get_nowait()
|
|
131
|
+
except Empty:
|
|
132
|
+
await asyncio.sleep(0.025)
|
|
133
|
+
continue
|
|
134
|
+
yield event
|
|
135
|
+
if event["type"] in {"result", "error"}:
|
|
136
|
+
return
|
|
137
|
+
finally:
|
|
138
|
+
# This does not pretend to interrupt an SDK network call. The worker
|
|
139
|
+
# checks the flag when that call returns, before any further DB work.
|
|
140
|
+
self.cancelled.set()
|
|
141
|
+
|
|
142
|
+
async def ndjson(self, request: Request | None = None) -> AsyncIterator[str]:
|
|
143
|
+
try:
|
|
144
|
+
async for event in self.events(request):
|
|
145
|
+
yield (
|
|
146
|
+
json.dumps(
|
|
147
|
+
{key: value for key, value in event.items() if key != "status"}, ensure_ascii=False, default=str
|
|
148
|
+
)
|
|
149
|
+
+ "\n"
|
|
150
|
+
)
|
|
151
|
+
finally:
|
|
152
|
+
self.cancelled.set()
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
async def analysis_response(conn_id: str, run: Analysis, request: Request) -> dict[str, Any] | Response:
|
|
156
|
+
operation = AnalysisOperation(conn_id, run)
|
|
157
|
+
if "application/x-ndjson" in request.headers.get("accept", ""):
|
|
158
|
+
# ASGI < 2.4 is watched by StreamingResponse itself. Newer ASGI
|
|
159
|
+
# relies on send errors, so poll disconnect while the model is silent.
|
|
160
|
+
spec = tuple(int(part) for part in request.scope.get("asgi", {}).get("spec_version", "2.0").split("."))
|
|
161
|
+
watched_request = request if spec >= (2, 4) else None
|
|
162
|
+
return StreamingResponse(
|
|
163
|
+
operation.ndjson(watched_request),
|
|
164
|
+
media_type="application/x-ndjson",
|
|
165
|
+
headers={"Cache-Control": "no-store", "X-Accel-Buffering": "no"},
|
|
166
|
+
)
|
|
167
|
+
async for event in operation.events(request):
|
|
168
|
+
if event["type"] == "result":
|
|
169
|
+
return dict(event["result"])
|
|
170
|
+
if event["type"] == "error":
|
|
171
|
+
raise HTTPException(event["status"], detail={"code": event["code"], "message": event["message"]})
|
|
172
|
+
raise HTTPException(499, detail={"code": "ai_cancelled", "message": "分析已取消。"})
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Bounded reads of current database records, independent of generation state."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from contextlib import closing, contextmanager
|
|
8
|
+
from dataclasses import asdict
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
from decimal import Decimal
|
|
11
|
+
from http import HTTPStatus
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from fastapi import APIRouter, HTTPException, Query
|
|
15
|
+
from fastapi.encoders import jsonable_encoder
|
|
16
|
+
from sqlseed._utils.paths import validate_table_name
|
|
17
|
+
from sqlseed._utils.sql_safe import quote_identifier
|
|
18
|
+
from sqlseed.database.sqlalchemy_adapter import SQLAlchemyAdapter
|
|
19
|
+
|
|
20
|
+
from sqlseed_web.state import ConnectionBusyError, UnknownConnectionError, state
|
|
21
|
+
from sqlseed_web.workbench_schema import _refresh_inspector, _target_identity
|
|
22
|
+
from sqlseed_web.workbench_store import get_store
|
|
23
|
+
|
|
24
|
+
router = APIRouter(prefix="/api/workbench", tags=["workbench-data"])
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@contextmanager
|
|
28
|
+
def _read_errors() -> Iterator[None]:
|
|
29
|
+
"""Keep driver exceptions, connection credentials and record values private."""
|
|
30
|
+
try:
|
|
31
|
+
yield
|
|
32
|
+
except HTTPException:
|
|
33
|
+
raise
|
|
34
|
+
except ConnectionBusyError as exc:
|
|
35
|
+
raise HTTPException(
|
|
36
|
+
409, detail={"code": "connection_busy", "message": "当前连接正在处理请求,请稍后重试。"}
|
|
37
|
+
) from exc
|
|
38
|
+
except KeyError as exc:
|
|
39
|
+
raise HTTPException(
|
|
40
|
+
404, detail={"code": "not_found", "message": "连接或运行记录已不存在,请刷新后重试。"}
|
|
41
|
+
) from exc
|
|
42
|
+
except Exception as exc:
|
|
43
|
+
raise HTTPException(
|
|
44
|
+
422,
|
|
45
|
+
detail={"code": "data_read_failed", "message": "读取数据失败,请检查连接、表结构和读取权限后重试。"},
|
|
46
|
+
) from exc
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _read_rows(
|
|
50
|
+
adapter: SQLAlchemyAdapter, table: str, order_by: list[str], limit: int, offset: int
|
|
51
|
+
) -> list[dict[str, Any]]:
|
|
52
|
+
"""Use the adapter's native parameter contract and release the pooled cursor."""
|
|
53
|
+
dialect = adapter.dialect.name
|
|
54
|
+
placeholder = {"sqlite": "?", "postgresql": "%s"}[dialect]
|
|
55
|
+
sql = f"SELECT * FROM {quote_identifier(table)}"
|
|
56
|
+
if order_by:
|
|
57
|
+
sql += " ORDER BY " + ", ".join(quote_identifier(column) for column in order_by)
|
|
58
|
+
# PostgreSQL DBAPI format/pyformat drivers parse percent signs even inside
|
|
59
|
+
# quoted names when parameters are supplied; literal percent signs double.
|
|
60
|
+
if dialect == "postgresql":
|
|
61
|
+
sql = sql.replace("%", "%%")
|
|
62
|
+
sql += f" LIMIT {placeholder} OFFSET {placeholder}"
|
|
63
|
+
with closing(adapter.execute(sql, (limit, offset))) as cursor:
|
|
64
|
+
names = [description[0] for description in cursor.description]
|
|
65
|
+
return [dict(zip(names, row, strict=True)) for row in cursor.fetchall()]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@router.get(
|
|
69
|
+
"/connections/{conn_id}/tables/{table:path}/data",
|
|
70
|
+
responses={
|
|
71
|
+
403: {"description": HTTPStatus(403).phrase},
|
|
72
|
+
404: {"description": HTTPStatus(404).phrase},
|
|
73
|
+
409: {"description": HTTPStatus(409).phrase},
|
|
74
|
+
422: {"description": HTTPStatus(422).phrase},
|
|
75
|
+
},
|
|
76
|
+
)
|
|
77
|
+
def table_data(
|
|
78
|
+
conn_id: str,
|
|
79
|
+
table: str,
|
|
80
|
+
limit: int = Query(default=50, ge=1, le=100),
|
|
81
|
+
offset: int = Query(default=0, ge=0),
|
|
82
|
+
run_id: str | None = Query(default=None, min_length=1, max_length=128),
|
|
83
|
+
) -> dict[str, Any]:
|
|
84
|
+
"""Read a current page; a run links the target and scope, never an insert delta."""
|
|
85
|
+
with _read_errors(), state.connection_operation(conn_id) as conn:
|
|
86
|
+
target_key, target_label = _target_identity(conn)
|
|
87
|
+
if run_id is not None:
|
|
88
|
+
run = get_store().get_run(run_id)
|
|
89
|
+
if run["target_key"] != target_key:
|
|
90
|
+
raise HTTPException(
|
|
91
|
+
409,
|
|
92
|
+
detail={"code": "target_mismatch", "message": "当前连接与运行记录的数据库不一致,请重新选择连接。"},
|
|
93
|
+
)
|
|
94
|
+
if table not in {item["name"] for item in run["tables"]}:
|
|
95
|
+
raise HTTPException(
|
|
96
|
+
403, detail={"code": "table_outside_run", "message": "该表不属于这条运行记录的生成范围。"}
|
|
97
|
+
)
|
|
98
|
+
conn.orchestrator.get_table_names() # Ensure the existing adapter is connected.
|
|
99
|
+
adapter = conn.orchestrator.database_adapter
|
|
100
|
+
if not isinstance(adapter, SQLAlchemyAdapter):
|
|
101
|
+
# An unsupported runtime adapter follows the existing RuntimeError API contract.
|
|
102
|
+
raise RuntimeError("Data browsing requires a SQLAlchemyAdapter connection") # noqa: TRY004
|
|
103
|
+
_refresh_inspector(conn, adapter)
|
|
104
|
+
names = adapter.get_table_names()
|
|
105
|
+
if table not in names:
|
|
106
|
+
raise HTTPException(
|
|
107
|
+
404, detail={"code": "table_not_found", "message": "该表已不存在,请重新读取数据库结构。"}
|
|
108
|
+
)
|
|
109
|
+
validate_table_name(table, names)
|
|
110
|
+
columns = [asdict(column) for column in adapter.get_column_info(table)]
|
|
111
|
+
order_by = adapter.get_primary_keys(table)
|
|
112
|
+
rows = _read_rows(adapter, table, order_by, limit, offset)
|
|
113
|
+
total = adapter.get_row_count(table)
|
|
114
|
+
result: dict[str, Any] = jsonable_encoder(
|
|
115
|
+
{
|
|
116
|
+
"table": table,
|
|
117
|
+
"target_key": target_key,
|
|
118
|
+
"target_label": target_label,
|
|
119
|
+
"dialect": adapter.dialect.name,
|
|
120
|
+
"columns": columns,
|
|
121
|
+
"rows": rows,
|
|
122
|
+
"total": total,
|
|
123
|
+
"limit": limit,
|
|
124
|
+
"offset": offset,
|
|
125
|
+
"order_by": order_by,
|
|
126
|
+
"read_at": datetime.now(timezone.utc).isoformat(),
|
|
127
|
+
},
|
|
128
|
+
custom_encoder={
|
|
129
|
+
bytes: lambda value: "0x" + value.hex(),
|
|
130
|
+
memoryview: lambda value: "0x" + value.hex(),
|
|
131
|
+
Decimal: str,
|
|
132
|
+
# JSON/browser numbers cannot represent these values faithfully.
|
|
133
|
+
int: lambda value: str(value) if abs(value) > 2**53 - 1 else value,
|
|
134
|
+
float: lambda value: str(value) if not math.isfinite(value) else value,
|
|
135
|
+
},
|
|
136
|
+
)
|
|
137
|
+
return result
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@router.get(
|
|
141
|
+
"/runs/{run_id}/data-connections",
|
|
142
|
+
responses={
|
|
143
|
+
404: {"description": HTTPStatus(404).phrase},
|
|
144
|
+
409: {"description": HTTPStatus(409).phrase},
|
|
145
|
+
422: {"description": HTTPStatus(422).phrase},
|
|
146
|
+
},
|
|
147
|
+
)
|
|
148
|
+
def run_data_connections(run_id: str) -> dict[str, Any]:
|
|
149
|
+
"""Match registered target identities without connecting or reflecting any DB."""
|
|
150
|
+
with _read_errors():
|
|
151
|
+
run = get_store().get_run(run_id)
|
|
152
|
+
connections = []
|
|
153
|
+
for item in state.list_connections():
|
|
154
|
+
try:
|
|
155
|
+
# Registry access takes its short lock. Identity uses immutable
|
|
156
|
+
# target metadata, so a busy database remains a valid candidate.
|
|
157
|
+
conn = state.get_connection(item["conn_id"])
|
|
158
|
+
except UnknownConnectionError:
|
|
159
|
+
continue
|
|
160
|
+
target_key, target_label = _target_identity(conn)
|
|
161
|
+
if target_key == run["target_key"]:
|
|
162
|
+
connections.append({"conn_id": conn.conn_id, "target_label": target_label})
|
|
163
|
+
return {"connections": connections, "target_key": run["target_key"], "target_label": run["target_label"]}
|