sqlseed-web 0.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlseed_web/AGENTS.md +106 -0
- sqlseed_web/__init__.py +24 -0
- sqlseed_web/__main__.py +8 -0
- sqlseed_web/_application.py +205 -0
- sqlseed_web/ai_settings.py +290 -0
- sqlseed_web/api.py +1102 -0
- sqlseed_web/app.py +26 -0
- sqlseed_web/managed_worker.py +184 -0
- sqlseed_web/operation_errors.py +42 -0
- sqlseed_web/plugin_environment.py +231 -0
- sqlseed_web/plugin_management.py +322 -0
- sqlseed_web/plugin_process.py +131 -0
- sqlseed_web/runtime_lifecycle.py +130 -0
- sqlseed_web/runtime_session.py +97 -0
- sqlseed_web/settings_environment.py +368 -0
- sqlseed_web/sqlite_target.py +86 -0
- sqlseed_web/state.py +348 -0
- sqlseed_web/static/AGENTS.md +180 -0
- sqlseed_web/static/ai.css +57 -0
- sqlseed_web/static/configs.css +50 -0
- sqlseed_web/static/date-picker.css +230 -0
- sqlseed_web/static/disclosure.css +149 -0
- sqlseed_web/static/graph-clarity.css +103 -0
- sqlseed_web/static/index.html +29 -0
- sqlseed_web/static/js/api.js +228 -0
- sqlseed_web/static/js/app.js +97 -0
- sqlseed_web/static/js/dropdown.js +432 -0
- sqlseed_web/static/js/filepicker.js +203 -0
- sqlseed_web/static/js/genform.js +1066 -0
- sqlseed_web/static/js/labels.js +195 -0
- sqlseed_web/static/js/pages/browse.js +209 -0
- sqlseed_web/static/js/pages/configs.js +424 -0
- sqlseed_web/static/js/pages/connect.js +332 -0
- sqlseed_web/static/js/pages/heal.js +395 -0
- sqlseed_web/static/js/pages/meta.js +110 -0
- sqlseed_web/static/js/pages/runs.js +293 -0
- sqlseed_web/static/js/pages/settings.js +942 -0
- sqlseed_web/static/js/pages/wizard.js +751 -0
- sqlseed_web/static/js/pages/workbench.js +3123 -0
- sqlseed_web/static/js/tree.js +126 -0
- sqlseed_web/static/js/workbench/ai-eligibility.js +33 -0
- sqlseed_web/static/js/workbench/ai-handoff.js +31 -0
- sqlseed_web/static/js/workbench/ai-stream.js +116 -0
- sqlseed_web/static/js/workbench/ai.js +888 -0
- sqlseed_web/static/js/workbench/connection.js +508 -0
- sqlseed_web/static/js/workbench/date-picker.js +445 -0
- sqlseed_web/static/js/workbench/dependency-view.js +119 -0
- sqlseed_web/static/js/workbench/editor.js +1236 -0
- sqlseed_web/static/js/workbench/focus.js +11 -0
- sqlseed_web/static/js/workbench/graph-layout.js +332 -0
- sqlseed_web/static/js/workbench/graph.js +970 -0
- sqlseed_web/static/js/workbench/guidance.js +29 -0
- sqlseed_web/static/js/workbench/model.js +124 -0
- sqlseed_web/static/js/workbench/plugin-management.js +512 -0
- sqlseed_web/static/js/workbench/preview-scroll-layout.js +94 -0
- sqlseed_web/static/js/workbench/preview.js +572 -0
- sqlseed_web/static/js/workbench/provider-guide.js +33 -0
- sqlseed_web/static/js/workbench/recovery.js +28 -0
- sqlseed_web/static/js/workbench/scroll-lock.js +26 -0
- sqlseed_web/static/js/workbench/session.js +174 -0
- sqlseed_web/static/js/workbench/table-data.js +186 -0
- sqlseed_web/static/js/workbench/ui.js +262 -0
- sqlseed_web/static/navigation.css +92 -0
- sqlseed_web/static/preview.css +29 -0
- sqlseed_web/static/runs.css +53 -0
- sqlseed_web/static/scrollbars.css +42 -0
- sqlseed_web/static/settings.css +108 -0
- sqlseed_web/static/style.css +3382 -0
- sqlseed_web/static/table-data.css +27 -0
- sqlseed_web/static/workbench.css +509 -0
- sqlseed_web/supervised_plugins.py +173 -0
- sqlseed_web/supervisor.py +238 -0
- sqlseed_web/workbench.py +381 -0
- sqlseed_web/workbench_ai.py +887 -0
- sqlseed_web/workbench_ai_relations.py +285 -0
- sqlseed_web/workbench_ai_stream.py +172 -0
- sqlseed_web/workbench_data.py +163 -0
- sqlseed_web/workbench_execution.py +199 -0
- sqlseed_web/workbench_runtime.py +1218 -0
- sqlseed_web/workbench_schema.py +277 -0
- sqlseed_web/workbench_store.py +458 -0
- sqlseed_web/worker_control.py +192 -0
- sqlseed_web-0.2.4.dist-info/METADATA +105 -0
- sqlseed_web-0.2.4.dist-info/RECORD +87 -0
- sqlseed_web-0.2.4.dist-info/WHEEL +4 -0
- sqlseed_web-0.2.4.dist-info/entry_points.txt +2 -0
- sqlseed_web-0.2.4.dist-info/licenses/LICENSE +679 -0
|
@@ -0,0 +1,1218 @@
|
|
|
1
|
+
"""Offline plan validation and server-owned multi-table execution for the workbench."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import re
|
|
9
|
+
import sqlite3
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import asdict, dataclass, replace
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import yaml
|
|
17
|
+
from sqlalchemy.engine import make_url
|
|
18
|
+
from sqlalchemy.exc import StatementError
|
|
19
|
+
from sqlseed._utils.logger import get_logger
|
|
20
|
+
from sqlseed._utils.sql_safe import quote_identifier
|
|
21
|
+
from sqlseed._utils.type_checks import has_exact_type
|
|
22
|
+
from sqlseed.config.models import (
|
|
23
|
+
ColumnAssociation,
|
|
24
|
+
ColumnConfig,
|
|
25
|
+
ColumnConstraintsConfig,
|
|
26
|
+
CustomColumnMappings,
|
|
27
|
+
ExactColumnMappingRule,
|
|
28
|
+
GeneratorConfig,
|
|
29
|
+
PatternColumnMappingRule,
|
|
30
|
+
TableConfig,
|
|
31
|
+
)
|
|
32
|
+
from sqlseed.core.column_dag import ColumnDAG
|
|
33
|
+
from sqlseed.core.expression import ExpressionEngine
|
|
34
|
+
from sqlseed.core.orchestrator import DataOrchestrator
|
|
35
|
+
from sqlseed.core.result import GenerationResult
|
|
36
|
+
from sqlseed.core.stream import GenerationBudgetExceededError, GenerationCancelledError
|
|
37
|
+
from sqlseed.database.sqlalchemy_adapter import SQLAlchemyAdapter
|
|
38
|
+
from sqlseed.generators._dispatch import GeneratorDispatchMixin
|
|
39
|
+
|
|
40
|
+
from sqlseed_web.operation_errors import generation_errors
|
|
41
|
+
from sqlseed_web.runtime_lifecycle import start_background
|
|
42
|
+
from sqlseed_web.settings_environment import package_availability
|
|
43
|
+
from sqlseed_web.state import Connection, UIState, state
|
|
44
|
+
from sqlseed_web.workbench_execution import build_execution_plan, normalize_execution
|
|
45
|
+
from sqlseed_web.workbench_schema import inspect_connection
|
|
46
|
+
from sqlseed_web.workbench_store import WorkspaceStore, get_store
|
|
47
|
+
|
|
48
|
+
_CREDENTIAL_KEY_PATTERN = r"password|passwd|pwd|secret|token|credential|key|passfile"
|
|
49
|
+
|
|
50
|
+
logger = get_logger(__name__)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class WorkbenchError(ValueError):
|
|
54
|
+
"""An actionable request error with a stable code and HTTP status."""
|
|
55
|
+
|
|
56
|
+
def __init__(self, message: str, *, code: str = "invalid_config", status: int = 422) -> None:
|
|
57
|
+
super().__init__(message)
|
|
58
|
+
self.code = code
|
|
59
|
+
self.status = status
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _redact_query_credentials(message: str) -> str:
|
|
63
|
+
"""Consume each query field once, including incomplete keys containing '?'."""
|
|
64
|
+
field_pattern = re.compile(r"[?&][^=&\s]*=?")
|
|
65
|
+
value_pattern = re.compile(r"[^&\s'\")]+")
|
|
66
|
+
parts: list[str] = []
|
|
67
|
+
cursor = copied = 0
|
|
68
|
+
while field := field_pattern.search(message, cursor):
|
|
69
|
+
cursor = field.end()
|
|
70
|
+
key = field.group()
|
|
71
|
+
if not key.endswith("=") or not re.search(_CREDENTIAL_KEY_PATTERN, key, re.IGNORECASE):
|
|
72
|
+
continue
|
|
73
|
+
if (value := value_pattern.match(message, cursor)) is not None:
|
|
74
|
+
parts.extend((message[copied:cursor], "***"))
|
|
75
|
+
cursor = copied = value.end()
|
|
76
|
+
return "".join(parts) + message[copied:]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _redact_url_credentials(message: str) -> str:
|
|
80
|
+
"""Scan each URL authority once, without regex backtracking on long errors."""
|
|
81
|
+
parts = message.split("://")
|
|
82
|
+
for index in range(1, len(parts)):
|
|
83
|
+
authority = parts[index]
|
|
84
|
+
for offset, character in enumerate(authority):
|
|
85
|
+
if character == "@":
|
|
86
|
+
if offset:
|
|
87
|
+
parts[index] = "***" + authority[offset:]
|
|
88
|
+
break
|
|
89
|
+
if character == "/" or character.isspace():
|
|
90
|
+
break
|
|
91
|
+
return "://".join(parts)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def public_error(exc: Exception) -> str:
|
|
95
|
+
"""Avoid exposing connection credentials or SQLAlchemy parameter dumps."""
|
|
96
|
+
if isinstance(exc, StatementError) and exc.orig is not None:
|
|
97
|
+
message = str(exc.orig)
|
|
98
|
+
else:
|
|
99
|
+
message = str(exc)
|
|
100
|
+
message = message.split("\n[SQL:", 1)[0].split("\n[parameters:", 1)[0]
|
|
101
|
+
message = _redact_url_credentials(message)
|
|
102
|
+
message = _redact_query_credentials(message)
|
|
103
|
+
return message[:2000]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _hash(value: Any) -> str:
|
|
107
|
+
return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False, default=str).encode()).hexdigest()
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _identity(target: str) -> str:
|
|
111
|
+
if "://" not in target:
|
|
112
|
+
return str(Path(target).expanduser().resolve())
|
|
113
|
+
url = make_url(target)
|
|
114
|
+
database = url.database
|
|
115
|
+
if url.get_backend_name() == "sqlite" and database and database != ":memory:":
|
|
116
|
+
return str(Path(database).expanduser().resolve())
|
|
117
|
+
secret_keys = [key for key in url.query if re.search(_CREDENTIAL_KEY_PATTERN, key, re.IGNORECASE)]
|
|
118
|
+
return url._replace(password=None).difference_update_query(secret_keys).render_as_string(hide_password=False)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _reject_extra(value: Any, fields: Any, path: str) -> None:
|
|
122
|
+
if isinstance(value, dict) and (extra := set(value) - set(fields)):
|
|
123
|
+
raise WorkbenchError(f"{path} 包含未知字段:{', '.join(sorted(extra))}", code="unknown_field")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _validate_table_keys(table: Any) -> None:
|
|
127
|
+
_reject_extra(table, TableConfig.model_fields, "table")
|
|
128
|
+
if not isinstance(table, dict):
|
|
129
|
+
return
|
|
130
|
+
for column in table.get("columns", []) or []:
|
|
131
|
+
if isinstance(column, dict):
|
|
132
|
+
_reject_extra(column.get("constraints"), ColumnConstraintsConfig.model_fields, "constraints")
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _validate_keys(raw: dict[str, Any]) -> None:
|
|
136
|
+
_reject_extra(raw, GeneratorConfig.model_fields, "配置")
|
|
137
|
+
for table in raw.get("tables", []) or []:
|
|
138
|
+
_validate_table_keys(table)
|
|
139
|
+
for association in raw.get("associations", []) or []:
|
|
140
|
+
_reject_extra(association, ColumnAssociation.model_fields, "association")
|
|
141
|
+
mappings = raw.get("custom_column_mappings")
|
|
142
|
+
_reject_extra(mappings, CustomColumnMappings.model_fields, "custom_column_mappings")
|
|
143
|
+
if isinstance(mappings, dict):
|
|
144
|
+
for rule in (mappings.get("exact") or {}).values():
|
|
145
|
+
_reject_extra(rule, ExactColumnMappingRule.model_fields, "custom mapping")
|
|
146
|
+
for rule in mappings.get("pattern", []) or []:
|
|
147
|
+
_reject_extra(rule, PatternColumnMappingRule.model_fields, "custom pattern")
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def bind_document(conn: Connection, document: dict[str, Any]) -> GeneratorConfig:
|
|
151
|
+
"""Validate the complete core model and bind only the explicitly selected target."""
|
|
152
|
+
_validate_keys(document)
|
|
153
|
+
db_path, url = document.get("db_path"), document.get("url")
|
|
154
|
+
if db_path is not None and url is not None:
|
|
155
|
+
raise WorkbenchError("db_path 与 url 只能提供一个", code="target_mismatch")
|
|
156
|
+
for supplied in (db_path, url):
|
|
157
|
+
if supplied is not None and _identity(str(supplied)) != _identity(conn.target):
|
|
158
|
+
raise WorkbenchError("配置目标与当前连接不匹配,请明确选择相同目标", code="target_mismatch", status=409)
|
|
159
|
+
raw = {key: value for key, value in document.items() if key not in {"db_path", "url"}}
|
|
160
|
+
raw.setdefault("provider", conn.provider)
|
|
161
|
+
raw.setdefault("locale", conn.locale)
|
|
162
|
+
raw["url" if "://" in conn.target else "db_path"] = conn.target
|
|
163
|
+
try:
|
|
164
|
+
return GeneratorConfig.model_validate(raw)
|
|
165
|
+
except (ValueError, TypeError) as exc:
|
|
166
|
+
raise WorkbenchError(public_error(exc)) from exc
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def normalize_document(conn: Connection, document: dict[str, Any]) -> dict[str, Any]:
|
|
170
|
+
"""Round-trip all core fields while omitting the connection and its credentials."""
|
|
171
|
+
return bind_document(conn, document).model_dump(mode="json", exclude={"db_path", "url"})
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def parse_document(conn: Connection, text: str) -> dict[str, Any]:
|
|
175
|
+
"""Parse safe YAML (including JSON), then apply core's configuration model."""
|
|
176
|
+
try:
|
|
177
|
+
raw = yaml.safe_load(text)
|
|
178
|
+
except yaml.YAMLError as exc:
|
|
179
|
+
raise WorkbenchError(f"无法解析 YAML/JSON:{public_error(exc)}", code="parse_error") from exc
|
|
180
|
+
if not isinstance(raw, dict):
|
|
181
|
+
raise WorkbenchError("配置必须是 YAML/JSON object", code="parse_error")
|
|
182
|
+
return normalize_document(conn, raw)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def export_document(conn: Connection, document: dict[str, Any]) -> dict[str, Any]:
|
|
186
|
+
"""Export an executable core config, with URL passwords and secret query options removed."""
|
|
187
|
+
config = bind_document(conn, document).model_dump(mode="json", exclude_none=True)
|
|
188
|
+
omitted = False
|
|
189
|
+
if config.get("url"):
|
|
190
|
+
url = make_url(config["url"])
|
|
191
|
+
secret_keys = [key for key in url.query if re.search(_CREDENTIAL_KEY_PATTERN, key, re.IGNORECASE)]
|
|
192
|
+
omitted = url.password is not None or bool(secret_keys)
|
|
193
|
+
config["url"] = (
|
|
194
|
+
url._replace(password=None).difference_update_query(secret_keys).render_as_string(hide_password=False)
|
|
195
|
+
)
|
|
196
|
+
return {
|
|
197
|
+
"yaml": yaml.safe_dump(config, allow_unicode=True, sort_keys=False),
|
|
198
|
+
"json": json.dumps(config, ensure_ascii=False, indent=2),
|
|
199
|
+
"credentials_omitted": omitted,
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _issue(issues: list[dict[str, Any]], code: str, message: str, *, severity: str = "error", **context: Any) -> None:
|
|
204
|
+
issues.append({"severity": severity, "code": code, "message": message, **context})
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _layers(dependencies: dict[str, set[str]]) -> tuple[list[str], list[list[str]]]:
|
|
208
|
+
pending = {name: set(parents) for name, parents in dependencies.items()}
|
|
209
|
+
layers: list[list[str]] = []
|
|
210
|
+
while pending:
|
|
211
|
+
if not (layer := [name for name, parents in pending.items() if not parents]):
|
|
212
|
+
break
|
|
213
|
+
layers.append(layer)
|
|
214
|
+
for name in layer:
|
|
215
|
+
del pending[name]
|
|
216
|
+
for parents in pending.values():
|
|
217
|
+
parents.difference_update(layer)
|
|
218
|
+
return [name for layer in layers for name in layer], layers
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _source_values(orch: DataOrchestrator, table: str, columns: list[str]) -> list[dict[str, Any]]:
|
|
222
|
+
quoted = [quote_identifier(column) for column in columns]
|
|
223
|
+
return orch.query(
|
|
224
|
+
f"SELECT DISTINCT {', '.join(quoted)} FROM {quote_identifier(table)} "
|
|
225
|
+
f"WHERE {' AND '.join(f'{column} IS NOT NULL' for column in quoted)} "
|
|
226
|
+
f"ORDER BY {', '.join(quoted)} LIMIT 10000"
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _can_omit(column: dict[str, Any]) -> bool:
|
|
231
|
+
return bool(
|
|
232
|
+
column.get("default") is not None
|
|
233
|
+
or column.get("is_autoincrement")
|
|
234
|
+
or column.get("is_computed")
|
|
235
|
+
or column.get("is_rowid_alias")
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _runtime_columns(config: GeneratorConfig, table: TableConfig, orch: DataOrchestrator) -> list[ColumnConfig]:
|
|
240
|
+
"""Protect matched custom rules from core's subsequent schema fallback.
|
|
241
|
+
|
|
242
|
+
Explicit per-table columns retain precedence. The mapper still selects the
|
|
243
|
+
applicable exact/pattern rule, including its normal name and type priority.
|
|
244
|
+
"""
|
|
245
|
+
columns = list(table.columns)
|
|
246
|
+
if (mappings := config.custom_column_mappings) is None:
|
|
247
|
+
return columns
|
|
248
|
+
configured = {column.name for column in columns}
|
|
249
|
+
for info in orch.get_column_info(table.name):
|
|
250
|
+
if info.name in configured:
|
|
251
|
+
continue
|
|
252
|
+
names = {info.name.lower(), orch._mapper._to_snake_case(info.name)}
|
|
253
|
+
candidates: list[Any] = [rule for name, rule in mappings.exact.items() if name.lower() in names]
|
|
254
|
+
candidates.extend(rule for rule in mappings.pattern if any(re.match(rule.pattern, name) for name in names))
|
|
255
|
+
mapped = orch.map_column(info)
|
|
256
|
+
if any(mapped.generator_name == rule.generator and mapped.params == rule.params for rule in candidates):
|
|
257
|
+
columns.append(
|
|
258
|
+
ColumnConfig(
|
|
259
|
+
name=info.name,
|
|
260
|
+
generator=mapped.generator_name,
|
|
261
|
+
params=dict(mapped.params),
|
|
262
|
+
null_ratio=mapped.null_ratio,
|
|
263
|
+
)
|
|
264
|
+
)
|
|
265
|
+
return columns
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _table_option_issues(table: TableConfig, issues: list[dict[str, Any]]) -> None:
|
|
269
|
+
if table.clear_before:
|
|
270
|
+
_issue(
|
|
271
|
+
issues,
|
|
272
|
+
"clear_not_supported",
|
|
273
|
+
"工作台尚未支持经过外键校验的清空计划;请关闭 clear_before",
|
|
274
|
+
table=table.name,
|
|
275
|
+
)
|
|
276
|
+
if table.transform:
|
|
277
|
+
_issue(
|
|
278
|
+
issues,
|
|
279
|
+
"transform_not_supported",
|
|
280
|
+
"工作台尚未支持执行服务器 Python transform;配置会保留在导出中",
|
|
281
|
+
table=table.name,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _unique_domain_issues(
|
|
286
|
+
column: ColumnConfig,
|
|
287
|
+
table: TableConfig,
|
|
288
|
+
metadata: dict[str, Any],
|
|
289
|
+
context: dict[str, str],
|
|
290
|
+
issues: list[dict[str, Any]],
|
|
291
|
+
) -> None:
|
|
292
|
+
single_unique = [constraint["columns"] for constraint in metadata["unique_constraints"]]
|
|
293
|
+
single_unique.append(metadata["primary_key"])
|
|
294
|
+
is_unique = [column.name] in single_unique or bool(column.constraints and column.constraints.unique)
|
|
295
|
+
if is_unique and column.null_ratio == 0:
|
|
296
|
+
choices = column.params.get("choices", column.params.get("weighted_choices"))
|
|
297
|
+
if (
|
|
298
|
+
column.generator in {"choice", "weighted_choice"}
|
|
299
|
+
and isinstance(choices, (list, dict))
|
|
300
|
+
and (available := len({_hash(value) for value in choices})) < table.count
|
|
301
|
+
):
|
|
302
|
+
_issue(
|
|
303
|
+
issues,
|
|
304
|
+
"unique_domain_exhausted",
|
|
305
|
+
f"显式候选值只有 {available} 个,无法生成 {table.count} 个唯一值",
|
|
306
|
+
**context,
|
|
307
|
+
)
|
|
308
|
+
minimum, maximum = column.params.get("min_value"), column.params.get("max_value")
|
|
309
|
+
if (
|
|
310
|
+
column.generator == "integer"
|
|
311
|
+
and isinstance(minimum, int)
|
|
312
|
+
and isinstance(maximum, int)
|
|
313
|
+
and maximum - minimum + 1 < table.count
|
|
314
|
+
):
|
|
315
|
+
_issue(issues, "unique_domain_exhausted", "显式整数范围不足以生成所需的唯一值", **context)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _derived_column_issues(
|
|
319
|
+
column: ColumnConfig, columns: dict[str, Any], context: dict[str, str], issues: list[dict[str, Any]]
|
|
320
|
+
) -> None:
|
|
321
|
+
sources = column.derive_from
|
|
322
|
+
for source in [sources] if isinstance(sources, str) else sources or []:
|
|
323
|
+
if source not in columns:
|
|
324
|
+
_issue(issues, "unknown_derive_source", f"派生来源列不存在:{source}", **context)
|
|
325
|
+
if column.expression:
|
|
326
|
+
try:
|
|
327
|
+
expression = ast.parse(column.expression, mode="eval")
|
|
328
|
+
allowed_calls = set(ExpressionEngine.SAFE_FUNCTIONS) | {"lookup"}
|
|
329
|
+
for call in ast.walk(expression):
|
|
330
|
+
if isinstance(call, ast.Call) and isinstance(call.func, ast.Name) and call.func.id not in allowed_calls:
|
|
331
|
+
_issue(
|
|
332
|
+
issues,
|
|
333
|
+
"unknown_expression_function",
|
|
334
|
+
f"不支持的 expression 函数:{call.func.id}",
|
|
335
|
+
**context,
|
|
336
|
+
)
|
|
337
|
+
except SyntaxError as exc:
|
|
338
|
+
_issue(issues, "invalid_expression", public_error(exc), **context)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _column_presence_issues(
|
|
342
|
+
column: ColumnConfig, info: dict[str, Any], context: dict[str, str], issues: list[dict[str, Any]]
|
|
343
|
+
) -> None:
|
|
344
|
+
if column.null_ratio > 0 and not info.get("nullable", True):
|
|
345
|
+
_issue(issues, "not_null", "NOT NULL 列不能设置 null_ratio > 0", **context)
|
|
346
|
+
can_skip = _can_omit(info)
|
|
347
|
+
if column.generator == "skip" and not info.get("nullable", True) and not can_skip:
|
|
348
|
+
_issue(issues, "required_column", "没有默认值的 NOT NULL 列不能跳过", **context)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _table_column_issues(
|
|
352
|
+
config: GeneratorConfig, table: TableConfig, metadata: dict[str, Any], known: set[str], issues: list[dict[str, Any]]
|
|
353
|
+
) -> None:
|
|
354
|
+
_table_option_issues(table, issues)
|
|
355
|
+
columns = {column["name"]: column for column in metadata["columns"]}
|
|
356
|
+
seen: set[str] = set()
|
|
357
|
+
for column in table.columns:
|
|
358
|
+
context = {"table": table.name, "column": column.name}
|
|
359
|
+
if column.name in seen:
|
|
360
|
+
_issue(issues, "duplicate_column", "列配置重复", **context)
|
|
361
|
+
seen.add(column.name)
|
|
362
|
+
if column.provider is not None and column.provider != config.provider:
|
|
363
|
+
_issue(
|
|
364
|
+
issues,
|
|
365
|
+
"column_provider_not_supported",
|
|
366
|
+
"当前 core 使用全局 provider;暂不支持不同的每列 provider",
|
|
367
|
+
**context,
|
|
368
|
+
)
|
|
369
|
+
if column.generator and column.generator not in known:
|
|
370
|
+
_issue(issues, "unknown_generator", f"未知 generator:{column.generator}", **context)
|
|
371
|
+
if (info := columns.get(column.name)) is None:
|
|
372
|
+
_issue(issues, "unknown_column", "列已不存在,请刷新 schema 并修正配置", **context)
|
|
373
|
+
continue
|
|
374
|
+
_unique_domain_issues(column, table, metadata, context, issues)
|
|
375
|
+
_column_presence_issues(column, info, context, issues)
|
|
376
|
+
_derived_column_issues(column, columns, context, issues)
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _column_issues(config: GeneratorConfig, tables: dict[str, Any], issues: list[dict[str, Any]]) -> None:
|
|
380
|
+
known = set(GeneratorDispatchMixin.GENERATOR_MAP) | {"skip", "foreign_key", "foreign_key_or_integer"}
|
|
381
|
+
for table in config.tables:
|
|
382
|
+
if table.name not in tables:
|
|
383
|
+
_issue(issues, "unknown_table", "数据表不存在", table=table.name)
|
|
384
|
+
continue
|
|
385
|
+
_table_column_issues(config, table, tables[table.name], known, issues)
|
|
386
|
+
if config.snapshot_dir:
|
|
387
|
+
_issue(
|
|
388
|
+
issues,
|
|
389
|
+
"snapshot_not_supported",
|
|
390
|
+
"工作台使用持久化运行记录,尚未支持 snapshot_dir 文件写出;配置会保留在导出中",
|
|
391
|
+
)
|
|
392
|
+
if config.custom_column_mappings:
|
|
393
|
+
mappings = config.custom_column_mappings
|
|
394
|
+
for generator in (
|
|
395
|
+
*[rule.generator for rule in mappings.exact.values()],
|
|
396
|
+
*[rule.generator for rule in mappings.pattern],
|
|
397
|
+
):
|
|
398
|
+
if generator not in known:
|
|
399
|
+
_issue(issues, "unknown_generator", f"自定义映射包含未知 generator:{generator}")
|
|
400
|
+
for rule in mappings.pattern:
|
|
401
|
+
try:
|
|
402
|
+
re.compile(rule.pattern)
|
|
403
|
+
except re.error as exc:
|
|
404
|
+
_issue(issues, "invalid_pattern", public_error(exc))
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _empty_source_issues(
|
|
408
|
+
context: dict[str, str],
|
|
409
|
+
columns: list[str],
|
|
410
|
+
nullable: bool,
|
|
411
|
+
selected: set[str],
|
|
412
|
+
deferred: set[str],
|
|
413
|
+
issues: list[dict[str, Any]],
|
|
414
|
+
) -> None:
|
|
415
|
+
parent, target = context["source_table"], context["table"]
|
|
416
|
+
if parent == target:
|
|
417
|
+
if not nullable:
|
|
418
|
+
_issue(issues, "self_reference_no_seed", "空表的 NOT NULL 自引用缺少有效初始父行", **context)
|
|
419
|
+
elif len(columns) > 1:
|
|
420
|
+
_issue(issues, "composite_self_reference", "当前 core 未保证组合自引用的第二阶段回填", **context)
|
|
421
|
+
else:
|
|
422
|
+
_issue(
|
|
423
|
+
issues,
|
|
424
|
+
"self_reference_backfill",
|
|
425
|
+
"可空自引用先生成 NULL,再由 core 回填已生成父行",
|
|
426
|
+
severity="warning",
|
|
427
|
+
**context,
|
|
428
|
+
)
|
|
429
|
+
elif parent in selected:
|
|
430
|
+
deferred.add(target)
|
|
431
|
+
_issue(
|
|
432
|
+
issues,
|
|
433
|
+
"preview_requires_parent",
|
|
434
|
+
"父表当前没有可用值;执行时先填父表,预览无法提供完整关联样例",
|
|
435
|
+
severity="warning",
|
|
436
|
+
**context,
|
|
437
|
+
)
|
|
438
|
+
elif nullable:
|
|
439
|
+
_issue(
|
|
440
|
+
issues,
|
|
441
|
+
"nullable_parent_empty",
|
|
442
|
+
"父表没有可用值,core 将生成 NULL 外键",
|
|
443
|
+
severity="warning",
|
|
444
|
+
**context,
|
|
445
|
+
)
|
|
446
|
+
else:
|
|
447
|
+
_issue(
|
|
448
|
+
issues,
|
|
449
|
+
"missing_parent_source",
|
|
450
|
+
"非空外键/关联没有可用来源,请将父表加入计划或先准备有效父行",
|
|
451
|
+
**context,
|
|
452
|
+
)
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _association_sources(
|
|
456
|
+
config: GeneratorConfig,
|
|
457
|
+
tables: dict[str, Any],
|
|
458
|
+
dependencies: dict[str, set[str]],
|
|
459
|
+
source: Callable[[str, str, list[str], bool, str], None],
|
|
460
|
+
issues: list[dict[str, Any]],
|
|
461
|
+
) -> None:
|
|
462
|
+
for association in config.associations:
|
|
463
|
+
if association.strategy != "shared_pool":
|
|
464
|
+
_issue(issues, "association_strategy", "当前 core 尚未区分 random 关联策略,请使用 shared_pool")
|
|
465
|
+
for name in association.target_tables:
|
|
466
|
+
if name not in tables or association.column_name not in {c["name"] for c in tables[name]["columns"]}:
|
|
467
|
+
_issue(
|
|
468
|
+
issues,
|
|
469
|
+
"invalid_association_target",
|
|
470
|
+
"关联目标表或列不存在",
|
|
471
|
+
table=name,
|
|
472
|
+
column=association.column_name,
|
|
473
|
+
)
|
|
474
|
+
elif name in dependencies:
|
|
475
|
+
source(
|
|
476
|
+
name,
|
|
477
|
+
association.source_table,
|
|
478
|
+
[association.source_column or association.column_name],
|
|
479
|
+
False,
|
|
480
|
+
association.column_name,
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _foreign_key_sources(
|
|
485
|
+
tables: dict[str, Any],
|
|
486
|
+
dependencies: dict[str, set[str]],
|
|
487
|
+
source: Callable[[str, str, list[str], bool, str], None],
|
|
488
|
+
issues: list[dict[str, Any]],
|
|
489
|
+
) -> None:
|
|
490
|
+
for name in dependencies:
|
|
491
|
+
for fk in tables[name]["foreign_keys"]:
|
|
492
|
+
if fk.get("ref_schema") not in (None, "", "public", "main"):
|
|
493
|
+
_issue(issues, "cross_schema_fk", "当前 core 尚未保证跨 schema 外键生成", table=name)
|
|
494
|
+
continue
|
|
495
|
+
if len(fk["columns"]) > 2:
|
|
496
|
+
_issue(issues, "composite_fk_width", "当前 core 尚未保证三列及以上组合外键的元组配对", table=name)
|
|
497
|
+
continue
|
|
498
|
+
source(name, fk["ref_table"], fk["ref_columns"], fk["nullable"], ",".join(fk["columns"]))
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _dependency_plan(
|
|
502
|
+
config: GeneratorConfig, schema: dict[str, Any], orch: DataOrchestrator, issues: list[dict[str, Any]]
|
|
503
|
+
) -> tuple[list[str], list[list[str]], set[str], dict[str, Any]]:
|
|
504
|
+
tables = {table["name"]: table for table in schema["tables"]}
|
|
505
|
+
selected = {table.name for table in config.tables}
|
|
506
|
+
dependencies = {table.name: set[str]() for table in config.tables if table.name in tables}
|
|
507
|
+
deferred: set[str] = set()
|
|
508
|
+
evidence: dict[str, Any] = {
|
|
509
|
+
"row_counts": {name: table["row_count"] for name, table in tables.items()},
|
|
510
|
+
"source_checks": [],
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
def source(target: str, parent: str, columns: list[str], nullable: bool, column: str) -> None:
|
|
514
|
+
context = {"table": target, "column": column, "source_table": parent}
|
|
515
|
+
if parent not in tables or any(name not in {c["name"] for c in tables[parent]["columns"]} for name in columns):
|
|
516
|
+
_issue(issues, "invalid_parent_source", "引用的来源表或列不存在", **context)
|
|
517
|
+
return
|
|
518
|
+
values = _source_values(orch, parent, columns)
|
|
519
|
+
evidence["source_checks"].append(
|
|
520
|
+
{
|
|
521
|
+
**context,
|
|
522
|
+
"source_columns": columns,
|
|
523
|
+
"has_values": bool(values),
|
|
524
|
+
"row_count": tables[parent]["row_count"],
|
|
525
|
+
"selected": parent in selected,
|
|
526
|
+
"nullable": nullable,
|
|
527
|
+
}
|
|
528
|
+
)
|
|
529
|
+
evidence[f"{parent}:{','.join(columns)}"] = _hash(values)
|
|
530
|
+
if parent != target and parent in dependencies:
|
|
531
|
+
dependencies[target].add(parent)
|
|
532
|
+
if not values:
|
|
533
|
+
_empty_source_issues(context, columns, nullable, selected, deferred, issues)
|
|
534
|
+
|
|
535
|
+
_foreign_key_sources(tables, dependencies, source, issues)
|
|
536
|
+
_association_sources(config, tables, dependencies, source, issues)
|
|
537
|
+
order, layers = _layers(dependencies)
|
|
538
|
+
if len(order) != len(dependencies):
|
|
539
|
+
_issue(issues, "cross_table_cycle", "跨表循环需要通用 backfill;当前工作台不能安全执行该计划")
|
|
540
|
+
return order, layers, deferred, evidence
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _sample_issues(
|
|
544
|
+
table: dict[str, Any],
|
|
545
|
+
samples: list[dict[str, Any]],
|
|
546
|
+
issues: list[dict[str, Any]],
|
|
547
|
+
*,
|
|
548
|
+
excluded: set[str] | None = None,
|
|
549
|
+
) -> None:
|
|
550
|
+
excluded = excluded or set()
|
|
551
|
+
for column in table["columns"]:
|
|
552
|
+
name = column["name"]
|
|
553
|
+
if name in excluded or column.get("nullable", True):
|
|
554
|
+
continue
|
|
555
|
+
can_skip = _can_omit(column)
|
|
556
|
+
if any(row.get(name) is None and (name in row or not can_skip) for row in samples):
|
|
557
|
+
_issue(issues, "not_null_sample", "实际生成的样例违反 NOT NULL 约束", table=table["name"], column=name)
|
|
558
|
+
unique_groups = [constraint["columns"] for constraint in table["unique_constraints"]]
|
|
559
|
+
if table["primary_key"]:
|
|
560
|
+
unique_groups.append(table["primary_key"])
|
|
561
|
+
for columns in unique_groups:
|
|
562
|
+
keys = [
|
|
563
|
+
_hash([row[column] for column in columns])
|
|
564
|
+
for row in samples
|
|
565
|
+
if all(row.get(column) is not None for column in columns)
|
|
566
|
+
]
|
|
567
|
+
if len(keys) != len(set(keys)):
|
|
568
|
+
_issue(
|
|
569
|
+
issues, "unique_sample", "实际生成的样例违反 UNIQUE 约束", table=table["name"], column=",".join(columns)
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
@dataclass(frozen=True)
|
|
574
|
+
class _PreviewOptions:
|
|
575
|
+
count: int
|
|
576
|
+
preview: bool
|
|
577
|
+
sample_max_attempts: int | None
|
|
578
|
+
cancel_check: Callable[[], None] | None
|
|
579
|
+
|
|
580
|
+
def guard(self) -> None:
|
|
581
|
+
if self.cancel_check is not None:
|
|
582
|
+
try:
|
|
583
|
+
self.cancel_check()
|
|
584
|
+
except Exception as exc:
|
|
585
|
+
raise GenerationCancelledError(exc) from exc
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
@dataclass(frozen=True)
|
|
589
|
+
class _SampleRules:
|
|
590
|
+
columns: list[ColumnConfig]
|
|
591
|
+
specs: dict[str, Any]
|
|
592
|
+
user_configs: dict[str, Any]
|
|
593
|
+
unique: set[str]
|
|
594
|
+
composite: list[list[str]]
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def _resolve_sample_rules(config: GeneratorConfig, table: TableConfig, orch: DataOrchestrator) -> _SampleRules:
|
|
598
|
+
columns = _runtime_columns(config, table, orch)
|
|
599
|
+
specs, user_configs, unique, composite = orch._resolve_specs(table.name, table.count, None, columns, table.enrich)
|
|
600
|
+
return _SampleRules(columns, specs, user_configs, unique, composite)
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def _preview_provider_available(config: GeneratorConfig, issues: list[dict[str, Any]]) -> bool:
|
|
604
|
+
if config.provider.value == "mimesis":
|
|
605
|
+
availability = package_availability("mimesis", "mimesis")
|
|
606
|
+
if not availability["available"]:
|
|
607
|
+
missing = availability["status"] == "not_installed"
|
|
608
|
+
_issue(
|
|
609
|
+
issues,
|
|
610
|
+
"provider_not_installed" if missing else "provider_import_error",
|
|
611
|
+
"当前配置使用 Mimesis,但尚未安装;请在插件页安装,或明确更改生成引擎后重新检查。"
|
|
612
|
+
if missing
|
|
613
|
+
else "当前配置使用 Mimesis,但组件加载异常;请在插件页查看修复指引,或明确更改生成引擎。",
|
|
614
|
+
component_id="mimesis",
|
|
615
|
+
recovery_action="install" if missing else "repair",
|
|
616
|
+
)
|
|
617
|
+
return False
|
|
618
|
+
return True
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _deferred_table_samples(
|
|
622
|
+
config: GeneratorConfig,
|
|
623
|
+
table: TableConfig,
|
|
624
|
+
metadata: dict[str, Any],
|
|
625
|
+
orch: DataOrchestrator,
|
|
626
|
+
rules: _SampleRules,
|
|
627
|
+
options: _PreviewOptions,
|
|
628
|
+
issues: list[dict[str, Any]],
|
|
629
|
+
) -> None:
|
|
630
|
+
# Validate the independent part of a deferred table without
|
|
631
|
+
# inventing the database-generated keys it will later consume.
|
|
632
|
+
blocked = {column for fk in metadata["foreign_keys"] for column in fk["columns"]}
|
|
633
|
+
blocked.update(
|
|
634
|
+
association.column_name for association in config.associations if table.name in association.target_tables
|
|
635
|
+
)
|
|
636
|
+
for node in ColumnDAG().build(rules.specs, rules.columns):
|
|
637
|
+
if any(dependency in blocked for dependency in node.depends_on):
|
|
638
|
+
blocked.add(node.name)
|
|
639
|
+
independent = {key: spec for key, spec in rules.specs.items() if key not in blocked}
|
|
640
|
+
independent_users = {key: value for key, value in rules.user_configs.items() if key not in blocked}
|
|
641
|
+
partial_stream = orch._build_stream(
|
|
642
|
+
independent,
|
|
643
|
+
independent_users,
|
|
644
|
+
rules.unique - blocked,
|
|
645
|
+
None,
|
|
646
|
+
table.seed,
|
|
647
|
+
table_name=table.name,
|
|
648
|
+
composite_unique=[columns for columns in rules.composite if not blocked.intersection(columns)],
|
|
649
|
+
max_attempts=options.sample_max_attempts,
|
|
650
|
+
cancel_check=options.cancel_check,
|
|
651
|
+
)
|
|
652
|
+
partial_samples = next(
|
|
653
|
+
partial_stream.generate(min(options.count, table.count), min(options.count, table.count)), []
|
|
654
|
+
)
|
|
655
|
+
_sample_issues(metadata, partial_samples, issues, excluded=blocked)
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
def _table_samples(
|
|
659
|
+
config: GeneratorConfig,
|
|
660
|
+
table: TableConfig,
|
|
661
|
+
tables: dict[str, Any],
|
|
662
|
+
orch: DataOrchestrator,
|
|
663
|
+
result: dict[str, Any],
|
|
664
|
+
options: _PreviewOptions,
|
|
665
|
+
deferred: bool,
|
|
666
|
+
) -> None:
|
|
667
|
+
issues = result["issues"]
|
|
668
|
+
name = table.name
|
|
669
|
+
rules = _resolve_sample_rules(config, table, orch)
|
|
670
|
+
options.guard()
|
|
671
|
+
result["effective_rules"][name] = {
|
|
672
|
+
key: {k: v for k, v in asdict(spec).items() if k != "params"}
|
|
673
|
+
| {"params": {k: v for k, v in spec.params.items() if not k.startswith("_")}}
|
|
674
|
+
for key, spec in rules.specs.items()
|
|
675
|
+
}
|
|
676
|
+
stream = orch._build_stream(
|
|
677
|
+
rules.specs,
|
|
678
|
+
rules.user_configs,
|
|
679
|
+
rules.unique,
|
|
680
|
+
None,
|
|
681
|
+
table.seed,
|
|
682
|
+
table_name=name,
|
|
683
|
+
composite_unique=rules.composite,
|
|
684
|
+
max_attempts=options.sample_max_attempts,
|
|
685
|
+
cancel_check=options.cancel_check,
|
|
686
|
+
)
|
|
687
|
+
if deferred:
|
|
688
|
+
_deferred_table_samples(config, table, tables[name], orch, rules, options, issues)
|
|
689
|
+
return
|
|
690
|
+
samples = next(stream.generate(min(options.count, table.count), min(options.count, table.count)), [])
|
|
691
|
+
_sample_issues(tables[name], samples, issues)
|
|
692
|
+
if options.preview:
|
|
693
|
+
result["samples"][name] = samples
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def _preview_tables(
|
|
697
|
+
config: GeneratorConfig,
|
|
698
|
+
tables: dict[str, Any],
|
|
699
|
+
order: list[str],
|
|
700
|
+
deferred: set[str],
|
|
701
|
+
orch: DataOrchestrator,
|
|
702
|
+
result: dict[str, Any],
|
|
703
|
+
options: _PreviewOptions,
|
|
704
|
+
) -> None:
|
|
705
|
+
issues = result["issues"]
|
|
706
|
+
configs = {table.name: table for table in config.tables}
|
|
707
|
+
for name in order:
|
|
708
|
+
options.guard()
|
|
709
|
+
table = configs[name]
|
|
710
|
+
try:
|
|
711
|
+
_table_samples(config, table, tables, orch, result, options, name in deferred)
|
|
712
|
+
except GenerationCancelledError:
|
|
713
|
+
raise
|
|
714
|
+
except GenerationBudgetExceededError as exc:
|
|
715
|
+
location = f"表 {name}"
|
|
716
|
+
if exc.column is not None:
|
|
717
|
+
location += f" 的列 {exc.column}(generator: {exc.generator})"
|
|
718
|
+
_issue(
|
|
719
|
+
issues,
|
|
720
|
+
"generation_invalid",
|
|
721
|
+
f"样例校验在{location}达到 {exc.limit} 次尝试上限,请检查唯一值空间或约束冲突。",
|
|
722
|
+
table=name,
|
|
723
|
+
column=exc.column,
|
|
724
|
+
generator=exc.generator,
|
|
725
|
+
attempt_limit=exc.limit,
|
|
726
|
+
)
|
|
727
|
+
except generation_errors(orch) as exc:
|
|
728
|
+
_issue(issues, "generation_invalid", public_error(exc), table=name)
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def _check_generation(
|
|
732
|
+
config: GeneratorConfig, schema: dict[str, Any], result: dict[str, Any], options: _PreviewOptions
|
|
733
|
+
) -> None:
|
|
734
|
+
issues = result["issues"]
|
|
735
|
+
if not config.tables:
|
|
736
|
+
_issue(issues, "empty_plan", "请至少选择一张需要生成的表")
|
|
737
|
+
if len({table.name for table in config.tables}) != len(config.tables):
|
|
738
|
+
_issue(issues, "duplicate_table", "同一计划不能重复配置同一张表")
|
|
739
|
+
tables = {table["name"]: table for table in schema["tables"]}
|
|
740
|
+
_column_issues(config, tables, issues)
|
|
741
|
+
normalized = config.model_dump(mode="json", exclude={"db_path", "url"})
|
|
742
|
+
orch: DataOrchestrator | None = None
|
|
743
|
+
try:
|
|
744
|
+
with DataOrchestrator.from_config(config) as orch:
|
|
745
|
+
options.guard()
|
|
746
|
+
if orch._provider_name != config.provider.value:
|
|
747
|
+
raise WorkbenchError(f"provider {config.provider.value} 不可用,不能使用降级 provider 代替")
|
|
748
|
+
orch._registry.get(config.provider.value).set_locale(config.locale)
|
|
749
|
+
order, layers, deferred, evidence = _dependency_plan(config, schema, orch, issues)
|
|
750
|
+
result.update(order=order, layers=layers, preview_complete=not deferred, sources=evidence["source_checks"])
|
|
751
|
+
result["config_hash"] = _hash(
|
|
752
|
+
{"document": normalized, "schema_hash": schema["schema_hash"], "sources": evidence}
|
|
753
|
+
)
|
|
754
|
+
if not any(issue["severity"] == "error" for issue in issues):
|
|
755
|
+
_preview_tables(config, tables, order, deferred, orch, result, options)
|
|
756
|
+
except GenerationCancelledError as exc:
|
|
757
|
+
raise exc.reason from None
|
|
758
|
+
except generation_errors(orch) as exc:
|
|
759
|
+
_issue(issues, "validation_failed", public_error(exc))
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def check_document(
|
|
763
|
+
conn: Connection,
|
|
764
|
+
document: dict[str, Any],
|
|
765
|
+
schema_hash: str,
|
|
766
|
+
*,
|
|
767
|
+
count: int = 3,
|
|
768
|
+
preview: bool = False,
|
|
769
|
+
sample_max_attempts: int | None = None,
|
|
770
|
+
cancel_check: Callable[[], None] | None = None,
|
|
771
|
+
) -> dict[str, Any]:
|
|
772
|
+
"""Validate against fresh DDL and real parent sources without inserting rows.
|
|
773
|
+
|
|
774
|
+
Optional sample budgets apply independently to each table stream. Cancellation
|
|
775
|
+
propagates the guard's original exception and never becomes a validation issue.
|
|
776
|
+
"""
|
|
777
|
+
if cancel_check is not None:
|
|
778
|
+
cancel_check()
|
|
779
|
+
if not 1 <= count <= 100:
|
|
780
|
+
raise WorkbenchError("预览 count 必须在 1–100 之间")
|
|
781
|
+
if sample_max_attempts is not None and (not has_exact_type(sample_max_attempts, int) or sample_max_attempts <= 0):
|
|
782
|
+
raise WorkbenchError("sample_max_attempts 必须为正整数或 None")
|
|
783
|
+
|
|
784
|
+
options = _PreviewOptions(count, preview, sample_max_attempts, cancel_check)
|
|
785
|
+
|
|
786
|
+
schema = inspect_connection(conn)
|
|
787
|
+
issues: list[dict[str, Any]] = []
|
|
788
|
+
result: dict[str, Any] = {
|
|
789
|
+
"ok": False,
|
|
790
|
+
"schema_hash": schema["schema_hash"],
|
|
791
|
+
"config_hash": "",
|
|
792
|
+
"issues": issues,
|
|
793
|
+
"order": [],
|
|
794
|
+
"layers": [],
|
|
795
|
+
"samples": {},
|
|
796
|
+
"effective_rules": {},
|
|
797
|
+
"sources": [],
|
|
798
|
+
"preview_complete": True,
|
|
799
|
+
}
|
|
800
|
+
if schema_hash != schema["schema_hash"]:
|
|
801
|
+
_issue(issues, "schema_changed", "数据库结构已变化,请刷新 schema 后重新检查")
|
|
802
|
+
try:
|
|
803
|
+
config = bind_document(conn, document)
|
|
804
|
+
except (WorkbenchError, TypeError, AttributeError) as exc:
|
|
805
|
+
_issue(issues, getattr(exc, "code", "invalid_config"), public_error(exc))
|
|
806
|
+
return result
|
|
807
|
+
if not _preview_provider_available(config, issues):
|
|
808
|
+
return result
|
|
809
|
+
_check_generation(config, schema, result, options)
|
|
810
|
+
if cancel_check is not None:
|
|
811
|
+
cancel_check()
|
|
812
|
+
result["ok"] = not any(issue["severity"] == "error" for issue in issues)
|
|
813
|
+
return result
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def _checked_saved(
|
|
817
|
+
conn: Connection, store: WorkspaceStore, draft_id: str, revision: int, schema_hash: str, config_hash: str
|
|
818
|
+
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
|
819
|
+
draft = store.get_draft(draft_id)
|
|
820
|
+
if draft["revision"] != revision:
|
|
821
|
+
raise WorkbenchError("草稿版本已变化,请重新保存并检查", code="revision_conflict", status=409)
|
|
822
|
+
schema = inspect_connection(conn)
|
|
823
|
+
if draft["target_key"] != schema["target_key"]:
|
|
824
|
+
raise WorkbenchError("草稿与当前连接目标不匹配", code="target_mismatch", status=409)
|
|
825
|
+
if draft["schema_hash"] != schema_hash:
|
|
826
|
+
raise WorkbenchError("已保存草稿的 schema 版本不匹配", code="schema_changed", status=409)
|
|
827
|
+
checked = check_document(conn, draft["document"], schema_hash)
|
|
828
|
+
if not checked["ok"]:
|
|
829
|
+
raise WorkbenchError(
|
|
830
|
+
"检查未通过:" + "; ".join(i["message"] for i in checked["issues"] if i["severity"] == "error"),
|
|
831
|
+
code="check_failed",
|
|
832
|
+
status=409,
|
|
833
|
+
)
|
|
834
|
+
if not config_hash or checked["config_hash"] != config_hash:
|
|
835
|
+
raise WorkbenchError("配置或数据来源已变化,请重新检查", code="check_stale", status=409)
|
|
836
|
+
return draft, schema, checked
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def _execution_options(execution: dict[str, Any] | None) -> dict[str, Any]:
|
|
840
|
+
try:
|
|
841
|
+
return normalize_execution(execution)
|
|
842
|
+
except ValueError as exc:
|
|
843
|
+
raise WorkbenchError(str(exc), code="invalid_execution") from exc
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def _require_execution_plan(plan: dict[str, Any], plan_hash: str) -> None:
|
|
847
|
+
if not plan["ok"]:
|
|
848
|
+
raise WorkbenchError(
|
|
849
|
+
"清空计划未通过:" + "; ".join(i["message"] for i in plan["issues"] if i["severity"] == "error"),
|
|
850
|
+
code="execution_blocked",
|
|
851
|
+
status=409,
|
|
852
|
+
)
|
|
853
|
+
if plan["mode"] == "replace_selected" and (not plan_hash or plan["plan_hash"] != plan_hash):
|
|
854
|
+
raise WorkbenchError("清空计划已变化,请重新预检并确认", code="execution_plan_stale", status=409)
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def plan_execution(
|
|
858
|
+
conn_id: str,
|
|
859
|
+
draft_id: str,
|
|
860
|
+
revision: int,
|
|
861
|
+
schema_hash: str,
|
|
862
|
+
config_hash: str,
|
|
863
|
+
*,
|
|
864
|
+
execution: dict[str, Any] | None = None,
|
|
865
|
+
registry: UIState = state,
|
|
866
|
+
store: WorkspaceStore | None = None,
|
|
867
|
+
) -> dict[str, Any]:
|
|
868
|
+
"""Read a saved/check-bound execution plan without changing the target."""
|
|
869
|
+
store = store or get_store()
|
|
870
|
+
options = _execution_options(execution)
|
|
871
|
+
with registry.connection_operation(conn_id) as conn:
|
|
872
|
+
draft, schema, checked = _checked_saved(conn, store, draft_id, revision, schema_hash, config_hash)
|
|
873
|
+
return build_execution_plan(
|
|
874
|
+
conn, bind_document(conn, draft["document"]), schema, checked["order"], options, config_hash
|
|
875
|
+
)
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def start_run(
|
|
879
|
+
conn_id: str,
|
|
880
|
+
draft_id: str,
|
|
881
|
+
revision: int,
|
|
882
|
+
schema_hash: str,
|
|
883
|
+
config_hash: str,
|
|
884
|
+
*,
|
|
885
|
+
execution: dict[str, Any] | None = None,
|
|
886
|
+
plan_hash: str = "",
|
|
887
|
+
registry: UIState = state,
|
|
888
|
+
store: WorkspaceStore | None = None,
|
|
889
|
+
) -> dict[str, Any]:
|
|
890
|
+
"""Reserve a live connection and execute only an unchanged, saved, checked revision."""
|
|
891
|
+
store = store or get_store()
|
|
892
|
+
options = _execution_options(execution)
|
|
893
|
+
with registry.connection_operation(conn_id, write=True) as conn:
|
|
894
|
+
draft, schema, checked = _checked_saved(conn, store, draft_id, revision, schema_hash, config_hash)
|
|
895
|
+
plan = build_execution_plan(
|
|
896
|
+
conn, bind_document(conn, draft["document"]), schema, checked["order"], options, config_hash
|
|
897
|
+
)
|
|
898
|
+
_require_execution_plan(plan, plan_hash)
|
|
899
|
+
tables_by_name = {table["name"]: table for table in draft["document"]["tables"]}
|
|
900
|
+
payload = {
|
|
901
|
+
"draft_id": draft_id,
|
|
902
|
+
"revision": revision,
|
|
903
|
+
"name": draft["name"],
|
|
904
|
+
"target_key": draft["target_key"],
|
|
905
|
+
"target_label": draft["target_label"],
|
|
906
|
+
"document": draft["document"],
|
|
907
|
+
"schema_hash": schema_hash,
|
|
908
|
+
"config_hash": config_hash,
|
|
909
|
+
"execution": options,
|
|
910
|
+
"plan_hash": plan["plan_hash"],
|
|
911
|
+
"status": "queued",
|
|
912
|
+
"order": checked["order"],
|
|
913
|
+
"rows_inserted": 0,
|
|
914
|
+
"errors": [],
|
|
915
|
+
"tables": [
|
|
916
|
+
{
|
|
917
|
+
"name": name,
|
|
918
|
+
"status": "queued",
|
|
919
|
+
"requested_count": tables_by_name[name]["count"],
|
|
920
|
+
"rows_inserted": 0,
|
|
921
|
+
"errors": [],
|
|
922
|
+
"batch_count": 0,
|
|
923
|
+
"elapsed": 0.0,
|
|
924
|
+
}
|
|
925
|
+
for name in checked["order"]
|
|
926
|
+
],
|
|
927
|
+
}
|
|
928
|
+
job = registry.create_job(conn_id, "workbench", draft["name"])
|
|
929
|
+
try:
|
|
930
|
+
run = store.create_run(payload, require_current_draft=True)
|
|
931
|
+
except Exception as exc:
|
|
932
|
+
# Reserve before writing the immutable record: another session
|
|
933
|
+
# targeting the same DB must not leave a duplicate queued record.
|
|
934
|
+
registry.complete_job(job.job_id, error=public_error(exc))
|
|
935
|
+
raise
|
|
936
|
+
try:
|
|
937
|
+
start_background(
|
|
938
|
+
target=execute_run,
|
|
939
|
+
args=(run["id"], conn_id, job.job_id),
|
|
940
|
+
kwargs={"registry": registry, "store": store},
|
|
941
|
+
daemon=True,
|
|
942
|
+
name=f"sqlseed-run-{run['id']}",
|
|
943
|
+
category="job",
|
|
944
|
+
)
|
|
945
|
+
except Exception as exc:
|
|
946
|
+
try:
|
|
947
|
+
store.update_run(run["id"], {"status": "error", "errors": [public_error(exc)], "finished_at": time.time()})
|
|
948
|
+
except (KeyError, ValueError, RuntimeError, OSError, sqlite3.Error):
|
|
949
|
+
logger.error("Failed to persist workbench startup failure", run_id=run["id"])
|
|
950
|
+
finally:
|
|
951
|
+
registry.complete_job(job.job_id, error=public_error(exc))
|
|
952
|
+
raise
|
|
953
|
+
return run
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def _fill_run_table(config: GeneratorConfig, table: TableConfig, orch: DataOrchestrator) -> GenerationResult:
|
|
957
|
+
return orch.fill_table(
|
|
958
|
+
table.name,
|
|
959
|
+
count=table.count,
|
|
960
|
+
batch_size=table.batch_size,
|
|
961
|
+
seed=table.seed,
|
|
962
|
+
column_configs=_runtime_columns(config, table, orch),
|
|
963
|
+
clear_before=False,
|
|
964
|
+
transform=None,
|
|
965
|
+
enrich=table.enrich,
|
|
966
|
+
skip_ai=True,
|
|
967
|
+
)
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def _replacement_plan(
|
|
971
|
+
conn: Connection, config: GeneratorConfig, run: dict[str, Any]
|
|
972
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
973
|
+
# BEGIN IMMEDIATE prevents another connection from changing the
|
|
974
|
+
# target between this final read and the destructive statements.
|
|
975
|
+
checked = check_document(conn, run["document"], run["schema_hash"])
|
|
976
|
+
if not checked["ok"] or checked["config_hash"] != run["config_hash"]:
|
|
977
|
+
raise WorkbenchError("执行前数据库来源或结构已变化,请重新检查与确认清空计划", code="check_stale")
|
|
978
|
+
schema = inspect_connection(conn)
|
|
979
|
+
plan = build_execution_plan(conn, config, schema, checked["order"], run["execution"], run["config_hash"])
|
|
980
|
+
_require_execution_plan(plan, run["plan_hash"])
|
|
981
|
+
return schema, plan
|
|
982
|
+
|
|
983
|
+
|
|
984
|
+
def _clear_replacement_tables(
|
|
985
|
+
adapter: SQLAlchemyAdapter, plan: dict[str, Any], schema: dict[str, Any], reset_identity: bool
|
|
986
|
+
) -> None:
|
|
987
|
+
for name in plan["delete_order"]:
|
|
988
|
+
adapter.execute(f"DELETE FROM {quote_identifier(name)}").close()
|
|
989
|
+
if reset_identity:
|
|
990
|
+
# Do not use the legacy dialect reset helper, which suppresses
|
|
991
|
+
# SQLite errors. A reset failure must roll back this whole run.
|
|
992
|
+
for table in schema["tables"]:
|
|
993
|
+
if table["name"] in plan["delete_order"] and any(column["is_autoincrement"] for column in table["columns"]):
|
|
994
|
+
adapter.execute("DELETE FROM sqlite_sequence WHERE name = ?", (table["name"],)).close()
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
def _fill_replacement_tables(
|
|
998
|
+
orch: DataOrchestrator,
|
|
999
|
+
config: GeneratorConfig,
|
|
1000
|
+
run: dict[str, Any],
|
|
1001
|
+
store: WorkspaceStore,
|
|
1002
|
+
tables: list[dict[str, Any]],
|
|
1003
|
+
) -> dict[str, dict[str, Any]]:
|
|
1004
|
+
configs = {table.name: table for table in config.tables}
|
|
1005
|
+
staged: dict[str, dict[str, Any]] = {}
|
|
1006
|
+
for item in tables:
|
|
1007
|
+
table = configs[item["name"]]
|
|
1008
|
+
item["status"] = "running"
|
|
1009
|
+
store.update_run(run["id"], {"tables": tables})
|
|
1010
|
+
result = _fill_run_table(config, table, orch)
|
|
1011
|
+
if result.errors:
|
|
1012
|
+
raise WorkbenchError("; ".join(public_error(ValueError(error)) for error in result.errors))
|
|
1013
|
+
staged[table.name] = {
|
|
1014
|
+
"rows_inserted": result.count,
|
|
1015
|
+
"batch_count": result.batch_count,
|
|
1016
|
+
"elapsed": result.elapsed,
|
|
1017
|
+
}
|
|
1018
|
+
if orch.query(f"PRAGMA foreign_key_check({quote_identifier(table.name)})"):
|
|
1019
|
+
raise WorkbenchError(f"{table.name} 生成后的外键检查未通过")
|
|
1020
|
+
return staged
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def _execute_replacement(
|
|
1024
|
+
conn: Connection,
|
|
1025
|
+
run: dict[str, Any],
|
|
1026
|
+
store: WorkspaceStore,
|
|
1027
|
+
tables: list[dict[str, Any]],
|
|
1028
|
+
outcome: dict[str, Any],
|
|
1029
|
+
) -> int:
|
|
1030
|
+
"""Keep every delete, sequence reset, FK read and streamed batch in one transaction."""
|
|
1031
|
+
config = bind_document(conn, run["document"])
|
|
1032
|
+
with DataOrchestrator.from_config(config) as orch:
|
|
1033
|
+
orch.get_table_names()
|
|
1034
|
+
adapter = orch.database_adapter
|
|
1035
|
+
if not isinstance(adapter, SQLAlchemyAdapter):
|
|
1036
|
+
raise WorkbenchError("清空生成需要 SQLAlchemyAdapter", code="execution_blocked")
|
|
1037
|
+
with adapter.transaction():
|
|
1038
|
+
outcome["rolled_back"] = True
|
|
1039
|
+
schema, plan = _replacement_plan(replace(conn, orchestrator=orch), config, run)
|
|
1040
|
+
_clear_replacement_tables(adapter, plan, schema, run["execution"]["reset_identity"])
|
|
1041
|
+
orch._relation.clear_cache()
|
|
1042
|
+
orch._shared_pool.clear()
|
|
1043
|
+
staged = _fill_replacement_tables(orch, config, run, store, tables)
|
|
1044
|
+
# Only the successful context exit above makes staged counts committed.
|
|
1045
|
+
outcome.update(committed=True, rolled_back=False)
|
|
1046
|
+
for item in tables:
|
|
1047
|
+
item.update(status="done", **staged[item["name"]])
|
|
1048
|
+
return sum(item["rows_inserted"] for item in tables)
|
|
1049
|
+
|
|
1050
|
+
|
|
1051
|
+
def _run_snapshot_failure(
|
|
1052
|
+
run_id: str, job_id: str, store: WorkspaceStore | None, registry: UIState, exc: Exception
|
|
1053
|
+
) -> None:
|
|
1054
|
+
# Failed snapshot loading must finish the reserved job, including storage failures.
|
|
1055
|
+
error = public_error(exc)
|
|
1056
|
+
try:
|
|
1057
|
+
if store is not None:
|
|
1058
|
+
store.update_run(run_id, {"status": "error", "errors": [error], "finished_at": time.time()})
|
|
1059
|
+
except (KeyError, ValueError, RuntimeError, OSError, sqlite3.Error):
|
|
1060
|
+
logger.error("Workbench run snapshot unavailable", run_id=run_id)
|
|
1061
|
+
finally:
|
|
1062
|
+
registry.complete_job(job_id, error=error)
|
|
1063
|
+
|
|
1064
|
+
|
|
1065
|
+
@dataclass
|
|
1066
|
+
class _RunProgress:
|
|
1067
|
+
rows_inserted: int
|
|
1068
|
+
errors: list[str]
|
|
1069
|
+
|
|
1070
|
+
|
|
1071
|
+
def _current_run_config(conn: Connection, run: dict[str, Any]) -> GeneratorConfig:
|
|
1072
|
+
checked = check_document(conn, run["document"], run["schema_hash"])
|
|
1073
|
+
if not checked["ok"] or checked["config_hash"] != run["config_hash"]:
|
|
1074
|
+
raise WorkbenchError("排队期间配置来源或 schema 已变化,运行未开始", code="check_stale")
|
|
1075
|
+
return bind_document(conn, run["document"])
|
|
1076
|
+
|
|
1077
|
+
|
|
1078
|
+
def _append_run_tables(
|
|
1079
|
+
config: GeneratorConfig, run_id: str, store: WorkspaceStore, tables: list[dict[str, Any]], progress: _RunProgress
|
|
1080
|
+
) -> None:
|
|
1081
|
+
configs = {table.name: table for table in config.tables}
|
|
1082
|
+
with DataOrchestrator.from_config(config) as orch:
|
|
1083
|
+
for item in tables:
|
|
1084
|
+
table = configs[item["name"]]
|
|
1085
|
+
item["status"] = "running"
|
|
1086
|
+
store.update_run(run_id, {"tables": tables})
|
|
1087
|
+
result = _fill_run_table(config, table, orch)
|
|
1088
|
+
progress.rows_inserted += result.count
|
|
1089
|
+
item.update(
|
|
1090
|
+
status="error" if result.errors else "done",
|
|
1091
|
+
rows_inserted=result.count,
|
|
1092
|
+
batch_count=result.batch_count,
|
|
1093
|
+
elapsed=result.elapsed,
|
|
1094
|
+
errors=[public_error(ValueError(error)) for error in result.errors],
|
|
1095
|
+
)
|
|
1096
|
+
store.update_run(run_id, {"tables": tables, "rows_inserted": progress.rows_inserted})
|
|
1097
|
+
if result.errors:
|
|
1098
|
+
progress.errors.extend(item["errors"])
|
|
1099
|
+
break
|
|
1100
|
+
|
|
1101
|
+
|
|
1102
|
+
def _run_execution_failure(
|
|
1103
|
+
exc: Exception, tables: list[dict[str, Any]], progress: _RunProgress, outcome: dict[str, Any], replacing: bool
|
|
1104
|
+
) -> None:
|
|
1105
|
+
# The worker boundary records sanitized failures while preserving committed counts.
|
|
1106
|
+
progress.errors.append(public_error(exc))
|
|
1107
|
+
if replacing and outcome["committed"]:
|
|
1108
|
+
# Disposal/reporting failures after COMMIT cannot erase rows that
|
|
1109
|
+
# are already committed or claim the transaction rolled back.
|
|
1110
|
+
progress.rows_inserted = sum(item["rows_inserted"] for item in tables)
|
|
1111
|
+
for item in tables:
|
|
1112
|
+
if item["status"] == "running":
|
|
1113
|
+
item.update(status="error", rows_inserted=0 if replacing else None, errors=[public_error(exc)])
|
|
1114
|
+
|
|
1115
|
+
|
|
1116
|
+
def _run_terminal(
|
|
1117
|
+
tables: list[dict[str, Any]], progress: _RunProgress, started: float, outcome: dict[str, Any], replacing: bool
|
|
1118
|
+
) -> dict[str, Any]:
|
|
1119
|
+
for item in tables:
|
|
1120
|
+
if item["status"] == "queued":
|
|
1121
|
+
item["status"] = "not_run"
|
|
1122
|
+
terminal: dict[str, Any] = {
|
|
1123
|
+
"status": "error" if progress.errors else "done",
|
|
1124
|
+
"tables": tables,
|
|
1125
|
+
"rows_inserted": progress.rows_inserted,
|
|
1126
|
+
"errors": progress.errors,
|
|
1127
|
+
"elapsed": time.monotonic() - started,
|
|
1128
|
+
"finished_at": time.time(),
|
|
1129
|
+
"row_counts_exact": all(item["rows_inserted"] is not None for item in tables),
|
|
1130
|
+
}
|
|
1131
|
+
if replacing:
|
|
1132
|
+
terminal["result"] = outcome
|
|
1133
|
+
return terminal
|
|
1134
|
+
|
|
1135
|
+
|
|
1136
|
+
def _publish_run_terminal(
|
|
1137
|
+
run_id: str,
|
|
1138
|
+
job_id: str,
|
|
1139
|
+
registry: UIState,
|
|
1140
|
+
store: WorkspaceStore,
|
|
1141
|
+
terminal: dict[str, Any],
|
|
1142
|
+
progress: _RunProgress,
|
|
1143
|
+
) -> None:
|
|
1144
|
+
persisted = False
|
|
1145
|
+
persistence_error = "运行终态保存意外中断,请检查服务日志。"
|
|
1146
|
+
try:
|
|
1147
|
+
store.update_run(run_id, terminal)
|
|
1148
|
+
persisted = True
|
|
1149
|
+
except (KeyError, ValueError, RuntimeError, OSError, sqlite3.Error) as exc:
|
|
1150
|
+
persistence_error = f"无法保存运行终态:{public_error(exc)}"
|
|
1151
|
+
finally:
|
|
1152
|
+
try:
|
|
1153
|
+
if not persisted:
|
|
1154
|
+
progress.errors.append(persistence_error)
|
|
1155
|
+
terminal["status"] = "error"
|
|
1156
|
+
logger.error("Failed to persist workbench run", run_id=run_id, error=persistence_error)
|
|
1157
|
+
_persist_failed_terminal(store, run_id, persistence_error)
|
|
1158
|
+
finally:
|
|
1159
|
+
registry.complete_job(
|
|
1160
|
+
job_id, result=terminal, error="; ".join(progress.errors) or None, rows_inserted=progress.rows_inserted
|
|
1161
|
+
)
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
def _persist_failed_terminal(store: WorkspaceStore, run_id: str, error: str) -> None:
|
|
1165
|
+
"""Try one minimal error record; the next startup recovers unavailable storage."""
|
|
1166
|
+
try:
|
|
1167
|
+
store.update_run(run_id, {"status": "error", "error": error, "errors": [error], "finished_at": time.time()})
|
|
1168
|
+
except (KeyError, ValueError, RuntimeError, OSError, sqlite3.Error):
|
|
1169
|
+
logger.error("Workbench run storage unavailable", run_id=run_id)
|
|
1170
|
+
|
|
1171
|
+
|
|
1172
|
+
def execute_run(
|
|
1173
|
+
run_id: str, conn_id: str, job_id: str, *, registry: UIState = state, store: WorkspaceStore | None = None
|
|
1174
|
+
) -> None:
|
|
1175
|
+
"""Run a frozen plan under the connection lock; persist each completed table."""
|
|
1176
|
+
with registry.job_completion(job_id):
|
|
1177
|
+
run = None
|
|
1178
|
+
try:
|
|
1179
|
+
store = store or get_store()
|
|
1180
|
+
run = store.get_run(run_id)
|
|
1181
|
+
except (KeyError, ValueError, RuntimeError, OSError, sqlite3.Error) as exc:
|
|
1182
|
+
_run_snapshot_failure(run_id, job_id, store, registry, exc)
|
|
1183
|
+
return
|
|
1184
|
+
finally:
|
|
1185
|
+
if run is None and registry.job_snapshot(job_id).status == "running":
|
|
1186
|
+
_run_snapshot_failure(run_id, job_id, store, registry, RuntimeError("运行快照读取意外中断。"))
|
|
1187
|
+
_execute_loaded_run(run, conn_id, job_id, registry, store)
|
|
1188
|
+
|
|
1189
|
+
|
|
1190
|
+
def _execute_loaded_run(
|
|
1191
|
+
run: dict[str, Any], conn_id: str, job_id: str, registry: UIState, store: WorkspaceStore
|
|
1192
|
+
) -> None:
|
|
1193
|
+
run_id = run["id"]
|
|
1194
|
+
tables = run["tables"]
|
|
1195
|
+
progress = _RunProgress(rows_inserted=0, errors=[])
|
|
1196
|
+
started = time.monotonic()
|
|
1197
|
+
replacing = run.get("execution", {}).get("mode") == "replace_selected"
|
|
1198
|
+
outcome: dict[str, Any] = {"atomic": replacing, "committed": False, "rolled_back": False}
|
|
1199
|
+
conn: Connection | None = None
|
|
1200
|
+
finished = False
|
|
1201
|
+
try:
|
|
1202
|
+
with registry.connection_operation(conn_id, job_id=job_id) as conn:
|
|
1203
|
+
config = _current_run_config(conn, run)
|
|
1204
|
+
store.update_run(run_id, {"status": "running", "started_at": time.time()})
|
|
1205
|
+
if replacing:
|
|
1206
|
+
progress.rows_inserted = _execute_replacement(conn, run, store, tables, outcome)
|
|
1207
|
+
else:
|
|
1208
|
+
_append_run_tables(config, run_id, store, tables, progress)
|
|
1209
|
+
finished = True
|
|
1210
|
+
except generation_errors(conn.orchestrator if conn is not None else None, additional=(KeyError,)) as exc:
|
|
1211
|
+
_run_execution_failure(exc, tables, progress, outcome, replacing)
|
|
1212
|
+
finished = True
|
|
1213
|
+
finally:
|
|
1214
|
+
if not finished:
|
|
1215
|
+
_run_execution_failure(RuntimeError("运行意外终止,请检查服务日志。"), tables, progress, outcome, replacing)
|
|
1216
|
+
_publish_run_terminal(
|
|
1217
|
+
run_id, job_id, registry, store, _run_terminal(tables, progress, started, outcome, replacing), progress
|
|
1218
|
+
)
|