eggpipe 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eggpipe/__init__.py +129 -0
- eggpipe/cli/__init__.py +0 -0
- eggpipe/cli/main.py +710 -0
- eggpipe/config/__init__.py +0 -0
- eggpipe/config/app_config.py +35 -0
- eggpipe/config/logging.py +40 -0
- eggpipe/config/project_config.py +162 -0
- eggpipe/core/__init__.py +0 -0
- eggpipe/core/composition.py +431 -0
- eggpipe/core/discovery.py +566 -0
- eggpipe/core/document.py +32 -0
- eggpipe/core/format_registry.py +230 -0
- eggpipe/core/layers.py +385 -0
- eggpipe/core/model.py +308 -0
- eggpipe/core/naming.py +36 -0
- eggpipe/core/parser.py +96 -0
- eggpipe/core/paths.py +12 -0
- eggpipe/core/provenance.py +52 -0
- eggpipe/core/slicing.py +110 -0
- eggpipe/core/validator.py +518 -0
- eggpipe/engine/__init__.py +0 -0
- eggpipe/engine/artifact_writer.py +741 -0
- eggpipe/engine/engine.py +679 -0
- eggpipe/engine/plugin_loader.py +215 -0
- eggpipe/engine/policy.py +27 -0
- eggpipe/engine/report.py +90 -0
- eggpipe/interfaces/__init__.py +0 -0
- eggpipe/interfaces/diagnostics.py +129 -0
- eggpipe/interfaces/formats.py +122 -0
- eggpipe/interfaces/generator.py +235 -0
- eggpipe/py.typed +0 -0
- eggpipe/templating.py +84 -0
- eggpipe/utils/__init__.py +0 -0
- eggpipe/utils/exceptions.py +65 -0
- eggpipe/utils/yaml.py +44 -0
- eggpipe/version.py +3 -0
- eggpipe/watch/__init__.py +0 -0
- eggpipe/watch/watcher.py +168 -0
- eggpipe-0.3.0.dist-info/METADATA +251 -0
- eggpipe-0.3.0.dist-info/RECORD +43 -0
- eggpipe-0.3.0.dist-info/WHEEL +4 -0
- eggpipe-0.3.0.dist-info/entry_points.txt +2 -0
- eggpipe-0.3.0.dist-info/licenses/LICENSE +21 -0
eggpipe/engine/engine.py
ADDED
|
@@ -0,0 +1,679 @@
|
|
|
1
|
+
"""Engine — library facade."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
import uuid
|
|
7
|
+
from collections.abc import Mapping
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from datetime import UTC, date, datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from eggpipe.config.app_config import AppConfig
|
|
13
|
+
from eggpipe.config.project_config import GeneratorConfig, ProjectConfig
|
|
14
|
+
from eggpipe.core.composition import Composition, CompositionResult
|
|
15
|
+
from eggpipe.core.discovery import SourceGroup, discover_sources
|
|
16
|
+
from eggpipe.core.document import ModelDocument
|
|
17
|
+
from eggpipe.core.format_registry import FormatRegistry
|
|
18
|
+
from eggpipe.core.model import PRIMITIVE_TYPES, FormatInfo, ModelSlice
|
|
19
|
+
from eggpipe.core.parser import Parser
|
|
20
|
+
from eggpipe.core.slicing import select_slice
|
|
21
|
+
from eggpipe.core.validator import validate_entity_model
|
|
22
|
+
from eggpipe.engine.artifact_writer import ArtifactWriter, ExecutionPlan, LocationKey
|
|
23
|
+
from eggpipe.engine.plugin_loader import build_format_registry, load_generators
|
|
24
|
+
from eggpipe.engine.policy import DiagnosticPolicy
|
|
25
|
+
from eggpipe.engine.report import (
|
|
26
|
+
ArtifactReport,
|
|
27
|
+
EngineReport,
|
|
28
|
+
FormatReport,
|
|
29
|
+
FormatSummary,
|
|
30
|
+
GeneratorReport,
|
|
31
|
+
ManifestLocationSummary,
|
|
32
|
+
ManifestSummary,
|
|
33
|
+
StaleLocation,
|
|
34
|
+
)
|
|
35
|
+
from eggpipe.interfaces.diagnostics import Diagnostic, error, warning
|
|
36
|
+
from eggpipe.interfaces.formats import EntityModelFormat
|
|
37
|
+
from eggpipe.interfaces.generator import (
|
|
38
|
+
GenerationContext,
|
|
39
|
+
GenerationResult,
|
|
40
|
+
GeneratorInput,
|
|
41
|
+
GeneratorPlugin,
|
|
42
|
+
ImportedFile,
|
|
43
|
+
)
|
|
44
|
+
from eggpipe.utils.exceptions import (
|
|
45
|
+
ConfigValidationError,
|
|
46
|
+
DocumentError,
|
|
47
|
+
LockError,
|
|
48
|
+
ManifestError,
|
|
49
|
+
)
|
|
50
|
+
from eggpipe.version import __version__
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class _Pending:
|
|
55
|
+
config: GeneratorConfig
|
|
56
|
+
plugin: GeneratorPlugin
|
|
57
|
+
result: GenerationResult
|
|
58
|
+
plan: ExecutionPlan
|
|
59
|
+
output_dir: Path
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class _GeneratorRun:
|
|
64
|
+
generators: tuple[GeneratorReport, ...]
|
|
65
|
+
artifacts: tuple[ArtifactReport, ...]
|
|
66
|
+
pending: tuple[_Pending, ...]
|
|
67
|
+
diagnostics: tuple[Diagnostic, ...]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class Engine:
|
|
71
|
+
def __init__(
|
|
72
|
+
self,
|
|
73
|
+
config: AppConfig,
|
|
74
|
+
rules: ProjectConfig,
|
|
75
|
+
*,
|
|
76
|
+
config_dir: Path | None = None,
|
|
77
|
+
generators: Mapping[str, GeneratorPlugin] | None = None,
|
|
78
|
+
formats: FormatRegistry | None = None,
|
|
79
|
+
source_paths: list[Path] | None = None,
|
|
80
|
+
variables: Mapping[str, str] | None = None,
|
|
81
|
+
) -> None:
|
|
82
|
+
self.config = config
|
|
83
|
+
self.rules = rules
|
|
84
|
+
self.config_dir = (config_dir or Path.cwd()).resolve()
|
|
85
|
+
self.registry = formats or build_format_registry()
|
|
86
|
+
self.parser = Parser(self.registry)
|
|
87
|
+
self.generators = dict(generators) if generators is not None else load_generators()
|
|
88
|
+
self.source_paths = source_paths
|
|
89
|
+
self.variables = {**rules.variables, **(variables or {})}
|
|
90
|
+
self.writer = ArtifactWriter(eggpipe_version=__version__, config_dir=self.config_dir)
|
|
91
|
+
self.policy = DiagnosticPolicy(self.rules.diagnostics.ignore)
|
|
92
|
+
self.run_id = uuid.uuid4().hex
|
|
93
|
+
self._pending: list[_Pending] = []
|
|
94
|
+
self._formats: list[FormatSummary] = []
|
|
95
|
+
self._stale: list[StaleLocation] = []
|
|
96
|
+
self._stale_paths: set[Path] = set()
|
|
97
|
+
self._timings: dict[str, int] = {}
|
|
98
|
+
self._started = 0.0
|
|
99
|
+
|
|
100
|
+
def build(self) -> EngineReport:
|
|
101
|
+
started = time.perf_counter()
|
|
102
|
+
self._started = started
|
|
103
|
+
self._timings = _empty_timings()
|
|
104
|
+
self._pending = []
|
|
105
|
+
self._formats = []
|
|
106
|
+
self._stale = []
|
|
107
|
+
self._stale_paths = set()
|
|
108
|
+
diagnostics: list[Diagnostic] = []
|
|
109
|
+
phase = time.perf_counter()
|
|
110
|
+
sources, source_groups, resources = self._source_paths(diagnostics)
|
|
111
|
+
self._timings["discovery_ms"] = _elapsed(phase)
|
|
112
|
+
if not sources:
|
|
113
|
+
return self._report(False, 1, diagnostics, [], started)
|
|
114
|
+
|
|
115
|
+
phase = time.perf_counter()
|
|
116
|
+
composition = self._load_composition(sources, source_groups, resources, diagnostics)
|
|
117
|
+
self._timings["parse_ms"] = _elapsed(phase)
|
|
118
|
+
if composition is None:
|
|
119
|
+
return self._report(False, 1, diagnostics, [], started)
|
|
120
|
+
|
|
121
|
+
phase = time.perf_counter()
|
|
122
|
+
self._formats = _format_summaries(composition)
|
|
123
|
+
self._validate_model(composition, diagnostics)
|
|
124
|
+
self._timings["validate_ms"] = _elapsed(phase)
|
|
125
|
+
diagnostics = self._filter_ignored(diagnostics)
|
|
126
|
+
if _has_errors(diagnostics):
|
|
127
|
+
return self._report(False, 1, diagnostics, [], started)
|
|
128
|
+
|
|
129
|
+
phase = time.perf_counter()
|
|
130
|
+
run = self._run_generators(composition)
|
|
131
|
+
self._timings["generate_ms"] = _elapsed(phase)
|
|
132
|
+
diagnostics.extend(run.diagnostics)
|
|
133
|
+
self._pending = list(run.pending)
|
|
134
|
+
if not any(item.code == "EG052" for item in diagnostics):
|
|
135
|
+
self._stale = self._stale_location_entries()
|
|
136
|
+
diagnostics.extend(_stale_diagnostics(self._stale))
|
|
137
|
+
success = not _has_errors(diagnostics)
|
|
138
|
+
return self._report(
|
|
139
|
+
success,
|
|
140
|
+
_exit_code(success, diagnostics),
|
|
141
|
+
self._filter_ignored(diagnostics),
|
|
142
|
+
list(run.generators),
|
|
143
|
+
started,
|
|
144
|
+
list(run.artifacts),
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
def _load_composition(
|
|
148
|
+
self,
|
|
149
|
+
sources: list[Path],
|
|
150
|
+
source_groups: list[SourceGroup] | None,
|
|
151
|
+
resources: dict[str, ImportedFile],
|
|
152
|
+
diagnostics: list[Diagnostic],
|
|
153
|
+
) -> CompositionResult | None:
|
|
154
|
+
composition = Composition(
|
|
155
|
+
self.parser,
|
|
156
|
+
project_root=self.config_dir,
|
|
157
|
+
allow_outside=self.config.allow_outside_imports,
|
|
158
|
+
)
|
|
159
|
+
try:
|
|
160
|
+
result = composition.resolve(sources, source_groups, resources=resources)
|
|
161
|
+
except DocumentError as exc:
|
|
162
|
+
diagnostics.append(error(exc.code, exc.message, file=exc.path))
|
|
163
|
+
return None
|
|
164
|
+
diagnostics.extend(result.diagnostics)
|
|
165
|
+
if self.rules.diagnostics.stale_dates:
|
|
166
|
+
diagnostics.extend(_stale_date_diagnostics(result.documents, self._stale_paths))
|
|
167
|
+
return result
|
|
168
|
+
|
|
169
|
+
def _validate_model(
|
|
170
|
+
self,
|
|
171
|
+
composition: CompositionResult,
|
|
172
|
+
diagnostics: list[Diagnostic],
|
|
173
|
+
) -> None:
|
|
174
|
+
plugins = [self.registry.get(document.format) for document in composition.documents]
|
|
175
|
+
primitives: set[str] = set()
|
|
176
|
+
for plugin in plugins:
|
|
177
|
+
primitives.update(plugin.primitive_types)
|
|
178
|
+
if not primitives:
|
|
179
|
+
primitives.update(PRIMITIVE_TYPES)
|
|
180
|
+
diagnostics.extend(
|
|
181
|
+
validate_entity_model(
|
|
182
|
+
composition.model,
|
|
183
|
+
primitive_types=primitives,
|
|
184
|
+
)
|
|
185
|
+
)
|
|
186
|
+
for document, plugin in zip(composition.documents, plugins, strict=True):
|
|
187
|
+
if isinstance(plugin, EntityModelFormat):
|
|
188
|
+
continue
|
|
189
|
+
diagnostics.extend(plugin.validate(document))
|
|
190
|
+
|
|
191
|
+
def _run_generators(self, composition: CompositionResult) -> _GeneratorRun:
|
|
192
|
+
generators: list[GeneratorReport] = []
|
|
193
|
+
artifacts: list[ArtifactReport] = []
|
|
194
|
+
pending: list[_Pending] = []
|
|
195
|
+
diagnostics: list[Diagnostic] = []
|
|
196
|
+
claimed: dict[str, LocationKey] = {}
|
|
197
|
+
for generator_config in self.rules.generators:
|
|
198
|
+
if not generator_config.enabled and not self.config.include_disabled:
|
|
199
|
+
continue
|
|
200
|
+
plugin = self.generators.get(generator_config.id)
|
|
201
|
+
if plugin is None:
|
|
202
|
+
raise ConfigValidationError(f"unknown generator: {generator_config.id}")
|
|
203
|
+
generator_started = time.perf_counter()
|
|
204
|
+
result, plan = self._run_generator(composition, generator_config, plugin)
|
|
205
|
+
plan.diagnostics.extend(
|
|
206
|
+
_claim_paths(
|
|
207
|
+
plan,
|
|
208
|
+
self.writer.location_key(
|
|
209
|
+
generator_config.id,
|
|
210
|
+
self._output_dir(generator_config),
|
|
211
|
+
),
|
|
212
|
+
claimed,
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
diagnostics.extend(result.warnings)
|
|
216
|
+
diagnostics.extend(result.errors)
|
|
217
|
+
diagnostics.extend(plan.diagnostics)
|
|
218
|
+
status = "ok" if result.success and not plan.has_errors else "error"
|
|
219
|
+
artifact_reports = [
|
|
220
|
+
ArtifactReport(
|
|
221
|
+
generator=generator_config.id,
|
|
222
|
+
path=str(item.path),
|
|
223
|
+
action=item.action,
|
|
224
|
+
sha256=item.sha256,
|
|
225
|
+
)
|
|
226
|
+
for item in plan.artifacts
|
|
227
|
+
]
|
|
228
|
+
artifacts.extend(artifact_reports)
|
|
229
|
+
generators.append(
|
|
230
|
+
GeneratorReport(
|
|
231
|
+
id=generator_config.id,
|
|
232
|
+
name=plugin.name or generator_config.id,
|
|
233
|
+
status=status,
|
|
234
|
+
input_formats=_input_formats(plugin),
|
|
235
|
+
formats=_format_reports(plan.formats),
|
|
236
|
+
duration_ms=int((time.perf_counter() - generator_started) * 1000),
|
|
237
|
+
)
|
|
238
|
+
)
|
|
239
|
+
if status == "ok":
|
|
240
|
+
pending.append(
|
|
241
|
+
_Pending(
|
|
242
|
+
config=generator_config,
|
|
243
|
+
plugin=plugin,
|
|
244
|
+
result=result,
|
|
245
|
+
plan=plan,
|
|
246
|
+
output_dir=self._output_dir(generator_config),
|
|
247
|
+
)
|
|
248
|
+
)
|
|
249
|
+
return _GeneratorRun(
|
|
250
|
+
generators=tuple(generators),
|
|
251
|
+
artifacts=tuple(artifacts),
|
|
252
|
+
pending=tuple(pending),
|
|
253
|
+
diagnostics=tuple(diagnostics),
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
def generate(self) -> EngineReport:
|
|
257
|
+
try:
|
|
258
|
+
with self.writer.write_lock():
|
|
259
|
+
report = self.build()
|
|
260
|
+
if not report.success:
|
|
261
|
+
return report
|
|
262
|
+
return self._write_pending(report)
|
|
263
|
+
except LockError as exc:
|
|
264
|
+
return self._report(
|
|
265
|
+
False,
|
|
266
|
+
1,
|
|
267
|
+
[error("EG300", f"failed to acquire the lock: {exc}")],
|
|
268
|
+
[],
|
|
269
|
+
time.perf_counter(),
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
def _write_pending(self, report: EngineReport) -> EngineReport:
|
|
273
|
+
diagnostics = list(report.diagnostics)
|
|
274
|
+
generators = list(report.generators)
|
|
275
|
+
phase = time.perf_counter()
|
|
276
|
+
try:
|
|
277
|
+
for pending in self._pending:
|
|
278
|
+
cleanup = "enabled" if self.config.clean else pending.config.cleanup
|
|
279
|
+
if cleanup == "prompt":
|
|
280
|
+
diagnostics.append(
|
|
281
|
+
warning(
|
|
282
|
+
"EG030",
|
|
283
|
+
"cleanup: prompt without a tty; using off",
|
|
284
|
+
path=str(pending.output_dir),
|
|
285
|
+
)
|
|
286
|
+
)
|
|
287
|
+
cleanup = "off"
|
|
288
|
+
diagnostics.extend(
|
|
289
|
+
self.writer.write(
|
|
290
|
+
generator_id=pending.config.id,
|
|
291
|
+
output_dir=pending.output_dir,
|
|
292
|
+
plan=pending.plan,
|
|
293
|
+
plugin_version=_plugin_version(pending.plugin),
|
|
294
|
+
cleanup=cleanup,
|
|
295
|
+
)
|
|
296
|
+
)
|
|
297
|
+
except (ManifestError, OSError) as exc:
|
|
298
|
+
diagnostics.append(error("EG300", f"failed to write artifacts: {exc}"))
|
|
299
|
+
if self._pending:
|
|
300
|
+
failed = {pending.config.id for pending in self._pending}
|
|
301
|
+
generators = [
|
|
302
|
+
item.model_copy(update={"status": "error"}) if item.id in failed else item
|
|
303
|
+
for item in generators
|
|
304
|
+
]
|
|
305
|
+
self._timings["write_ms"] = _elapsed(phase)
|
|
306
|
+
success = not _has_errors(diagnostics)
|
|
307
|
+
return report.model_copy(
|
|
308
|
+
update={
|
|
309
|
+
"success": success,
|
|
310
|
+
"exit_code": 0 if success else 1,
|
|
311
|
+
"diagnostics": diagnostics,
|
|
312
|
+
"generators": generators,
|
|
313
|
+
"manifest": self._manifest_summary(),
|
|
314
|
+
"timings": self._timings_with_total(),
|
|
315
|
+
}
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
def check(self) -> EngineReport:
|
|
319
|
+
report = self.build()
|
|
320
|
+
diagnostics = list(report.diagnostics)
|
|
321
|
+
for pending in self._pending:
|
|
322
|
+
diagnostics.extend(self.writer.check(plan=pending.plan))
|
|
323
|
+
success = not _has_errors(diagnostics)
|
|
324
|
+
return report.model_copy(
|
|
325
|
+
update={
|
|
326
|
+
"success": success,
|
|
327
|
+
"exit_code": _exit_code(success, diagnostics),
|
|
328
|
+
"diagnostics": diagnostics,
|
|
329
|
+
}
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
def _stale_location_entries(self) -> list[StaleLocation]:
|
|
333
|
+
try:
|
|
334
|
+
manifest = self.writer.load_manifest()
|
|
335
|
+
except ManifestError:
|
|
336
|
+
return []
|
|
337
|
+
configured = {
|
|
338
|
+
self.writer.location_key(item.id, self._output_dir(item))
|
|
339
|
+
for item in self.rules.generators
|
|
340
|
+
}
|
|
341
|
+
return [
|
|
342
|
+
StaleLocation(generator=location.generator, output_dir=location.output_dir)
|
|
343
|
+
for location in manifest.locations
|
|
344
|
+
if location.key not in configured
|
|
345
|
+
]
|
|
346
|
+
|
|
347
|
+
def _source_paths(
|
|
348
|
+
self, diagnostics: list[Diagnostic]
|
|
349
|
+
) -> tuple[list[Path], list[SourceGroup] | None, dict[str, ImportedFile]]:
|
|
350
|
+
if self.source_paths is not None:
|
|
351
|
+
return (
|
|
352
|
+
[
|
|
353
|
+
path if path.is_absolute() else (self.config_dir / path).resolve()
|
|
354
|
+
for path in self.source_paths
|
|
355
|
+
],
|
|
356
|
+
None,
|
|
357
|
+
{},
|
|
358
|
+
)
|
|
359
|
+
if not self.rules.sources:
|
|
360
|
+
raise ConfigValidationError(".eggpipe.yaml has no sources")
|
|
361
|
+
discovered = discover_sources(
|
|
362
|
+
self.rules,
|
|
363
|
+
self.config_dir,
|
|
364
|
+
allow_outside=self.config.allow_outside_imports,
|
|
365
|
+
)
|
|
366
|
+
diagnostics.extend(discovered.diagnostics)
|
|
367
|
+
for group in discovered.groups:
|
|
368
|
+
for unit in group.units or ():
|
|
369
|
+
if unit.stale_dates:
|
|
370
|
+
self._stale_paths.update(unit.paths)
|
|
371
|
+
return discovered.paths, discovered.groups, discovered.resources
|
|
372
|
+
|
|
373
|
+
def _run_generator(
|
|
374
|
+
self,
|
|
375
|
+
composition: CompositionResult,
|
|
376
|
+
generator_config: GeneratorConfig,
|
|
377
|
+
plugin: GeneratorPlugin,
|
|
378
|
+
) -> tuple[GenerationResult, ExecutionPlan]:
|
|
379
|
+
slice_, slice_diagnostics = select_slice(composition.model, generator_config.input)
|
|
380
|
+
context = GenerationContext(
|
|
381
|
+
variables=dict(self.variables),
|
|
382
|
+
naming_hint=self.rules.defaults.naming_convention,
|
|
383
|
+
imports=dict(composition.imports),
|
|
384
|
+
sources=tuple(_composition_sources(composition)),
|
|
385
|
+
formats=slice_.formats,
|
|
386
|
+
run_id=self.run_id,
|
|
387
|
+
config_dir=self.config_dir,
|
|
388
|
+
)
|
|
389
|
+
slice_errors, slice_warnings = _partition(slice_diagnostics)
|
|
390
|
+
if slice_errors:
|
|
391
|
+
return GenerationResult(warnings=slice_warnings, errors=slice_errors), ExecutionPlan()
|
|
392
|
+
|
|
393
|
+
format_errors = _check_input_formats(plugin, generator_config.id, slice_, self.registry)
|
|
394
|
+
if format_errors:
|
|
395
|
+
return GenerationResult(warnings=slice_warnings, errors=format_errors), ExecutionPlan()
|
|
396
|
+
|
|
397
|
+
try:
|
|
398
|
+
config_diagnostics = plugin.validate_config(generator_config)
|
|
399
|
+
except Exception as exc:
|
|
400
|
+
return (
|
|
401
|
+
GenerationResult(
|
|
402
|
+
warnings=slice_warnings,
|
|
403
|
+
errors=[
|
|
404
|
+
error(
|
|
405
|
+
"EG300",
|
|
406
|
+
f"validate_config {generator_config.id} failed: "
|
|
407
|
+
f"{type(exc).__name__}: {exc}",
|
|
408
|
+
)
|
|
409
|
+
],
|
|
410
|
+
),
|
|
411
|
+
ExecutionPlan(),
|
|
412
|
+
)
|
|
413
|
+
config_errors, config_warnings = _partition(config_diagnostics)
|
|
414
|
+
warnings = slice_warnings + config_warnings
|
|
415
|
+
if config_errors:
|
|
416
|
+
return GenerationResult(warnings=warnings, errors=config_errors), ExecutionPlan()
|
|
417
|
+
|
|
418
|
+
try:
|
|
419
|
+
result = plugin.generate(
|
|
420
|
+
GeneratorInput(model=slice_, config=generator_config, context=context)
|
|
421
|
+
)
|
|
422
|
+
except Exception as exc:
|
|
423
|
+
return (
|
|
424
|
+
GenerationResult(
|
|
425
|
+
warnings=warnings,
|
|
426
|
+
errors=[
|
|
427
|
+
error(
|
|
428
|
+
"EG300",
|
|
429
|
+
f"generator {generator_config.id} failed: {type(exc).__name__}: {exc}",
|
|
430
|
+
)
|
|
431
|
+
],
|
|
432
|
+
),
|
|
433
|
+
ExecutionPlan(),
|
|
434
|
+
)
|
|
435
|
+
if warnings:
|
|
436
|
+
result = result.model_copy(update={"warnings": [*warnings, *result.warnings]})
|
|
437
|
+
if not result.success:
|
|
438
|
+
return result, ExecutionPlan()
|
|
439
|
+
try:
|
|
440
|
+
plan = self.writer.plan(
|
|
441
|
+
generator_id=generator_config.id,
|
|
442
|
+
output_dir=self._output_dir(generator_config),
|
|
443
|
+
artifacts=result.artifacts,
|
|
444
|
+
on_conflict=generator_config.on_conflict,
|
|
445
|
+
force=self.config.force,
|
|
446
|
+
formats=slice_.formats,
|
|
447
|
+
)
|
|
448
|
+
except ManifestError as exc:
|
|
449
|
+
return (
|
|
450
|
+
GenerationResult(
|
|
451
|
+
warnings=warnings,
|
|
452
|
+
errors=[error("EG052", str(exc))],
|
|
453
|
+
),
|
|
454
|
+
ExecutionPlan(),
|
|
455
|
+
)
|
|
456
|
+
return result, plan
|
|
457
|
+
|
|
458
|
+
def _output_dir(self, generator_config: GeneratorConfig) -> Path:
|
|
459
|
+
base = self.config.output_root or self.config_dir
|
|
460
|
+
return (base / generator_config.output_dir).resolve()
|
|
461
|
+
|
|
462
|
+
def _filter_ignored(self, diagnostics: list[Diagnostic]) -> list[Diagnostic]:
|
|
463
|
+
return self.policy.apply(diagnostics)
|
|
464
|
+
|
|
465
|
+
def _report(
|
|
466
|
+
self,
|
|
467
|
+
success: bool,
|
|
468
|
+
exit_code: int,
|
|
469
|
+
diagnostics: list[Diagnostic],
|
|
470
|
+
generators: list[GeneratorReport],
|
|
471
|
+
started: float,
|
|
472
|
+
artifacts: list[ArtifactReport] | None = None,
|
|
473
|
+
) -> EngineReport:
|
|
474
|
+
return EngineReport(
|
|
475
|
+
success=success,
|
|
476
|
+
exit_code=exit_code,
|
|
477
|
+
run_id=self.run_id,
|
|
478
|
+
diagnostics=diagnostics,
|
|
479
|
+
formats=list(self._formats),
|
|
480
|
+
generators=generators,
|
|
481
|
+
artifacts=artifacts or [],
|
|
482
|
+
manifest=self._manifest_summary(),
|
|
483
|
+
stale_locations=list(self._stale),
|
|
484
|
+
timings=self._timings_with_total(),
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
def _timings_with_total(self) -> dict[str, int]:
|
|
488
|
+
return {
|
|
489
|
+
**self._timings,
|
|
490
|
+
"total_ms": _elapsed(self._started),
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
def _manifest_summary(self) -> ManifestSummary | None:
|
|
494
|
+
try:
|
|
495
|
+
manifest = self.writer.load_manifest()
|
|
496
|
+
except ManifestError:
|
|
497
|
+
return None
|
|
498
|
+
return ManifestSummary(
|
|
499
|
+
manifest_version=manifest.manifest_version,
|
|
500
|
+
locations=[
|
|
501
|
+
ManifestLocationSummary(
|
|
502
|
+
generator=location.generator,
|
|
503
|
+
output_dir=location.output_dir,
|
|
504
|
+
artifacts=len(location.artifacts),
|
|
505
|
+
)
|
|
506
|
+
for location in manifest.locations
|
|
507
|
+
],
|
|
508
|
+
)
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _empty_timings() -> dict[str, int]:
|
|
512
|
+
return {
|
|
513
|
+
"discovery_ms": 0,
|
|
514
|
+
"parse_ms": 0,
|
|
515
|
+
"validate_ms": 0,
|
|
516
|
+
"generate_ms": 0,
|
|
517
|
+
"write_ms": 0,
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _elapsed(started: float) -> int:
|
|
522
|
+
return int((time.perf_counter() - started) * 1000)
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def _plugin_version(plugin: GeneratorPlugin) -> str:
|
|
526
|
+
return str(getattr(plugin, "__version__", "unknown"))
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _stale_date_diagnostics(
|
|
530
|
+
documents: tuple[ModelDocument, ...],
|
|
531
|
+
allowed_paths: set[Path],
|
|
532
|
+
) -> list[Diagnostic]:
|
|
533
|
+
diagnostics: list[Diagnostic] = []
|
|
534
|
+
for document in documents:
|
|
535
|
+
if document.path not in allowed_paths:
|
|
536
|
+
continue
|
|
537
|
+
try:
|
|
538
|
+
modified = document.path.stat().st_mtime
|
|
539
|
+
except OSError:
|
|
540
|
+
continue
|
|
541
|
+
file_date = datetime.fromtimestamp(modified, tz=UTC).date()
|
|
542
|
+
for entity in document.model.entities:
|
|
543
|
+
if entity.updated_at is None:
|
|
544
|
+
continue
|
|
545
|
+
updated: date = (
|
|
546
|
+
entity.updated_at.date()
|
|
547
|
+
if isinstance(entity.updated_at, datetime)
|
|
548
|
+
else entity.updated_at
|
|
549
|
+
)
|
|
550
|
+
if updated < file_date:
|
|
551
|
+
diagnostics.append(
|
|
552
|
+
warning(
|
|
553
|
+
"EG014",
|
|
554
|
+
f"updated_at {updated.isoformat()} is older than "
|
|
555
|
+
f"mtime {file_date.isoformat()}",
|
|
556
|
+
file=document.path,
|
|
557
|
+
entity=entity.name,
|
|
558
|
+
)
|
|
559
|
+
)
|
|
560
|
+
return diagnostics
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def _composition_sources(composition: CompositionResult) -> list[Path]:
|
|
564
|
+
paths = {document.path for document in composition.documents}
|
|
565
|
+
for document in composition.documents:
|
|
566
|
+
for entity in document.model.entities:
|
|
567
|
+
paths.update(entity.layer_sources)
|
|
568
|
+
return sorted(paths, key=lambda item: item.as_posix())
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _format_summaries(composition: CompositionResult) -> list[FormatSummary]:
|
|
572
|
+
counts: dict[tuple[str, str], int] = {}
|
|
573
|
+
for entity in composition.model.entities:
|
|
574
|
+
if entity.format is None or entity.format_version is None:
|
|
575
|
+
continue
|
|
576
|
+
key = (entity.format, entity.format_version)
|
|
577
|
+
counts[key] = counts.get(key, 0) + 1
|
|
578
|
+
return [
|
|
579
|
+
FormatSummary(name=name, version=version, entities=count)
|
|
580
|
+
for (name, version), count in sorted(counts.items())
|
|
581
|
+
]
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _format_reports(formats: frozenset[FormatInfo]) -> list[FormatReport]:
|
|
585
|
+
return [
|
|
586
|
+
FormatReport(name=item.name, version=item.version)
|
|
587
|
+
for item in sorted(formats, key=lambda item: (item.name, item.version))
|
|
588
|
+
]
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def _input_formats(plugin: GeneratorPlugin) -> list[str] | None:
|
|
592
|
+
formats = getattr(type(plugin), "input_formats", None)
|
|
593
|
+
return None if formats is None else sorted(formats)
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _stale_diagnostics(stale: list[StaleLocation]) -> list[Diagnostic]:
|
|
597
|
+
return [
|
|
598
|
+
warning(
|
|
599
|
+
"EG051",
|
|
600
|
+
"stale generated location in the manifest",
|
|
601
|
+
path=item.output_dir,
|
|
602
|
+
)
|
|
603
|
+
for item in stale
|
|
604
|
+
]
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
def _claim_paths(
|
|
608
|
+
plan: ExecutionPlan,
|
|
609
|
+
location: LocationKey,
|
|
610
|
+
claimed: dict[str, LocationKey],
|
|
611
|
+
) -> list[Diagnostic]:
|
|
612
|
+
diagnostics: list[Diagnostic] = []
|
|
613
|
+
for item in plan.artifacts:
|
|
614
|
+
name = item.target.resolve().as_posix().casefold()
|
|
615
|
+
for other, owner in claimed.items():
|
|
616
|
+
if owner == location:
|
|
617
|
+
continue
|
|
618
|
+
if name == other or name.startswith(f"{other}/") or other.startswith(f"{name}/"):
|
|
619
|
+
diagnostics.append(
|
|
620
|
+
error(
|
|
621
|
+
"EG042",
|
|
622
|
+
f"path {item.path} is already claimed by location "
|
|
623
|
+
f"{owner.generator}:{owner.output_dir}",
|
|
624
|
+
path=str(item.path),
|
|
625
|
+
)
|
|
626
|
+
)
|
|
627
|
+
break
|
|
628
|
+
claimed[name] = location
|
|
629
|
+
return diagnostics
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def _check_input_formats(
|
|
633
|
+
plugin: GeneratorPlugin,
|
|
634
|
+
generator_id: str,
|
|
635
|
+
slice_: ModelSlice,
|
|
636
|
+
registry: FormatRegistry,
|
|
637
|
+
) -> list[Diagnostic]:
|
|
638
|
+
if not slice_.entities:
|
|
639
|
+
return []
|
|
640
|
+
missing = object()
|
|
641
|
+
accepted = getattr(type(plugin), "input_formats", missing)
|
|
642
|
+
if accepted is missing:
|
|
643
|
+
return [error("EG301", f"generator {generator_id}: input_formats is required")]
|
|
644
|
+
if accepted is not None and not isinstance(accepted, set):
|
|
645
|
+
return [error("EG301", f"generator {generator_id}: input_formats must be set[str]")]
|
|
646
|
+
errors: list[Diagnostic] = []
|
|
647
|
+
for entity in slice_.entities:
|
|
648
|
+
if entity.format is None:
|
|
649
|
+
continue
|
|
650
|
+
if not registry.accepts(entity.format, accepted):
|
|
651
|
+
errors.append(
|
|
652
|
+
error(
|
|
653
|
+
"EG033",
|
|
654
|
+
f"generator {generator_id} does not support format {entity.format}",
|
|
655
|
+
entity=entity.name,
|
|
656
|
+
)
|
|
657
|
+
)
|
|
658
|
+
return errors
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
def _partition(
|
|
662
|
+
diagnostics: list[Diagnostic],
|
|
663
|
+
) -> tuple[list[Diagnostic], list[Diagnostic]]:
|
|
664
|
+
errors = [item for item in diagnostics if item.level == "error"]
|
|
665
|
+
warnings = [item for item in diagnostics if item.level == "warning"]
|
|
666
|
+
return errors, warnings
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def _has_errors(diagnostics: list[Diagnostic]) -> bool:
|
|
670
|
+
return any(item.level == "error" for item in diagnostics)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def _exit_code(success: bool, diagnostics: list[Diagnostic]) -> int:
|
|
674
|
+
if success:
|
|
675
|
+
return 0
|
|
676
|
+
return 2 if any(item.code == "EG033" for item in diagnostics) else 1
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
__all__ = ["Engine"]
|