nativegate 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. nativegate/__init__.py +1 -0
  2. nativegate/__main__.py +4 -0
  3. nativegate/buildinfo.py +344 -0
  4. nativegate/cli.py +2007 -0
  5. nativegate/config.py +991 -0
  6. nativegate/declared_invariants.py +565 -0
  7. nativegate/discovery.py +167 -0
  8. nativegate/driverbuild.py +626 -0
  9. nativegate/drivers/__init__.py +5 -0
  10. nativegate/drivers/cpp.py +616 -0
  11. nativegate/drivers/fortran.py +507 -0
  12. nativegate/generators/__init__.py +0 -0
  13. nativegate/generators/cmake_gen.py +101 -0
  14. nativegate/generators/docker_gen.py +614 -0
  15. nativegate/generators/error_gen.py +104 -0
  16. nativegate/generators/f2py_gen.py +91 -0
  17. nativegate/generators/gateway_gen.py +110 -0
  18. nativegate/generators/golden_gen.py +50 -0
  19. nativegate/generators/k8s_gen.py +212 -0
  20. nativegate/generators/mcp_gen.py +281 -0
  21. nativegate/generators/middleware_gen.py +717 -0
  22. nativegate/generators/pybind_gen.py +406 -0
  23. nativegate/generators/pyproject_gen.py +61 -0
  24. nativegate/generators/python_pkg_gen.py +1164 -0
  25. nativegate/generators/test_gen.py +160 -0
  26. nativegate/golden.py +747 -0
  27. nativegate/invariants.py +532 -0
  28. nativegate/ir.py +789 -0
  29. nativegate/lattice.py +350 -0
  30. nativegate/locking.py +216 -0
  31. nativegate/oracle.py +904 -0
  32. nativegate/parsers/__init__.py +0 -0
  33. nativegate/parsers/cpp.py +105 -0
  34. nativegate/parsers/cpp_ast.py +1652 -0
  35. nativegate/parsers/cpp_regex.py +812 -0
  36. nativegate/parsers/fixed_form.py +868 -0
  37. nativegate/parsers/fortran.py +157 -0
  38. nativegate/parsers/fortran_fparser.py +1116 -0
  39. nativegate/parsers/fortran_regex.py +686 -0
  40. nativegate/preprocess.py +335 -0
  41. nativegate/structural_invariants.py +762 -0
  42. nativegate/suggest.py +208 -0
  43. nativegate/templates/__init__.py +20 -0
  44. nativegate/templates/golden_test_template.py +248 -0
  45. nativegate/wire.py +438 -0
  46. nativegate-0.1.0.dist-info/METADATA +547 -0
  47. nativegate-0.1.0.dist-info/RECORD +50 -0
  48. nativegate-0.1.0.dist-info/WHEEL +5 -0
  49. nativegate-0.1.0.dist-info/entry_points.txt +3 -0
  50. nativegate-0.1.0.dist-info/top_level.txt +1 -0
nativegate/cli.py ADDED
@@ -0,0 +1,2007 @@
1
+ """nativegate CLI (design.md section 13).
2
+
3
+ MVP 1 + MVP 2 scope: C++ / pybind11 / CMake and Fortran / f2py / CMake,
4
+ both end-to-end through `generate` and `build`.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import shutil
11
+ import subprocess
12
+ import sys
13
+ from pathlib import Path
14
+
15
+ import click
16
+ from rich import box
17
+ from rich.console import Console
18
+ from rich.progress import Progress, SpinnerColumn, TextColumn
19
+ from rich.table import Table
20
+
21
+ from .config import ClangConfig, ConfigError, ExposeConfig, ExposeWarning, ServiceConfig
22
+ from .discovery import (
23
+ detect_language,
24
+ find_cpp_directives,
25
+ find_implementation_files,
26
+ find_native_sources,
27
+ is_fixed_form,
28
+ requires_preprocessing,
29
+ )
30
+ from .preprocess import (
31
+ IncludeError,
32
+ PreprocessError,
33
+ run_c_preprocessor,
34
+ expand_includes,
35
+ read_source,
36
+ resolve_kind_parameters,
37
+ uses_kind_parameters,
38
+ )
39
+ from . import golden as golden_lib
40
+ from . import invariants as invariants_lib
41
+ from . import locking
42
+ from . import oracle as oracle_lib
43
+ from . import structural_invariants as si_lib
44
+ from .suggest import POOR, READY, WORKABLE, analyse_tree
45
+ from .generators import (
46
+ cmake_gen,
47
+ docker_gen,
48
+ f2py_gen,
49
+ middleware_gen,
50
+ gateway_gen,
51
+ golden_gen,
52
+ k8s_gen,
53
+ mcp_gen,
54
+ pybind_gen,
55
+ pyproject_gen,
56
+ python_pkg_gen,
57
+ test_gen,
58
+ )
59
+ from .ir import (
60
+ IRSchemaError,
61
+ ModuleIR,
62
+ NativeTypeError,
63
+ module_from_dict,
64
+ module_to_dict,
65
+ validate as validate_ir,
66
+ )
67
+ from .parsers import cpp as cpp_parser
68
+ from .parsers.cpp_ast import ClangOptions
69
+ from .parsers import fixed_form
70
+ from .parsers import fortran as fortran_parser
71
+
72
+ SERVICES_DIR = "services"
73
+
74
+
75
+ class StepTracker:
76
+ """Live terminal progress for a fixed sequence of steps.
77
+
78
+ Each step shows a spinner while running, then flips to a checkmark (or a
79
+ red X on failure) and stays on screen — so `quickstart` reads as a
80
+ checklist filling in rather than a wall of scrolled-past log lines.
81
+ """
82
+
83
+ def __init__(self, steps: list[str]) -> None:
84
+ self._console = Console()
85
+ self._progress = Progress(
86
+ SpinnerColumn(finished_text="[green]✔[/green]"),
87
+ TextColumn("{task.description}"),
88
+ console=self._console,
89
+ )
90
+ self._tasks = {}
91
+ self._steps = steps
92
+
93
+ def __enter__(self) -> "StepTracker":
94
+ self._progress.__enter__()
95
+ for step in self._steps:
96
+ self._tasks[step] = self._progress.add_task(step, total=1, start=False)
97
+ return self
98
+
99
+ def __exit__(self, exc_type, exc, tb) -> None:
100
+ self._progress.__exit__(exc_type, exc, tb)
101
+
102
+ def start(self, step: str) -> None:
103
+ self._progress.start_task(self._tasks[step])
104
+
105
+ def done(self, step: str) -> None:
106
+ self._progress.update(self._tasks[step], completed=1)
107
+ self._progress.stop_task(self._tasks[step])
108
+
109
+ def fail(self, step: str) -> None:
110
+ task = self._tasks[step]
111
+ self._progress.update(task, description=f"[red]✘ {self._progress.tasks[task].description}[/red]")
112
+ self._progress.stop_task(task)
113
+
114
+ def log(self, message: str) -> None:
115
+ self._console.print(f" [dim]{message}[/dim]")
116
+
117
+
118
+ def _load_config(service_dir: Path) -> ServiceConfig:
119
+ """ServiceConfig.load with its two new failure channels reported properly.
120
+
121
+ `language:` is now inferred from the sources actually present, so a config
122
+ that disagrees with native/ raises ConfigError — which reached the user as
123
+ a raw traceback — and a service exposing nothing by default now warns.
124
+ """
125
+ import warnings
126
+
127
+ with warnings.catch_warnings(record=True) as caught:
128
+ warnings.simplefilter("always", ExposeWarning)
129
+ try:
130
+ config = ServiceConfig.load(service_dir)
131
+ except ConfigError as exc:
132
+ raise click.ClickException(str(exc)) from exc
133
+
134
+ for warning in caught:
135
+ if issubclass(warning.category, ExposeWarning):
136
+ click.echo(f"WARNING: {warning.message}")
137
+ return config
138
+
139
+
140
+ def _symbol_names(module) -> list[str]:
141
+ if module is None:
142
+ return []
143
+ return (
144
+ [c.name for c in getattr(module, "classes", [])]
145
+ + [s.name for s in getattr(module, "structs", [])]
146
+ + [f.name for f in getattr(module, "functions", [])]
147
+ + [
148
+ m.name
149
+ for c in getattr(module, "classes", [])
150
+ for m in getattr(c, "methods", [])
151
+ ]
152
+ )
153
+
154
+
155
+ def _write_python(path: Path, source: str, module=None) -> None:
156
+ """Syntax-check generated Python, then write it (D3).
157
+
158
+ A generated file that will not parse has to fail the *build*. Before this
159
+ gate the only thing that ever compiled generated output was one golden
160
+ test, which is how a Fortran `&` line continuation leaked into a committed
161
+ router.py and shipped as a SyntaxError that surfaced at container start.
162
+ """
163
+ try:
164
+ compile(source, str(path), "exec")
165
+ except SyntaxError as exc:
166
+ offending = (exc.text or "").rstrip()
167
+ # Name the symbol whose codegen most likely produced the bad line, so
168
+ # the report points at the native declaration rather than at a line
169
+ # number in a file nobody wrote by hand.
170
+ implicated = sorted(
171
+ {name for name in _symbol_names(module) if name and name in offending}
172
+ )
173
+ detail = f" (symbol: {', '.join(implicated)})" if implicated else ""
174
+ raise click.ClickException(
175
+ f"nativegate generated invalid Python in {path} at line {exc.lineno}: "
176
+ f"{exc.msg}{detail}\n"
177
+ f" {offending}\n"
178
+ "This is a nativegate bug — the file was not written. Please report it "
179
+ "with the native declaration above."
180
+ ) from exc
181
+ path.write_text(source)
182
+
183
+
184
+ def _validate_module(module) -> None:
185
+ """Refuse to generate from an IR that cannot produce working Python (B4)."""
186
+ problems = validate_ir(module)
187
+ if not problems:
188
+ return
189
+ lines = "\n".join(f" - {p.symbol}: {p.message}" for p in problems)
190
+ raise click.ClickException(
191
+ f"Cannot generate a Python package for '{module.name}':\n{lines}"
192
+ )
193
+
194
+
195
+ def _service_dir(name: str) -> Path:
196
+ path = Path(SERVICES_DIR) / name
197
+ if not path.exists():
198
+ raise click.ClickException(
199
+ f"No service '{name}' found at {path}. Run `ngate create-service {name}` first."
200
+ )
201
+ return path
202
+
203
+
204
+ @click.group()
205
+ @click.version_option()
206
+ def main() -> None:
207
+ """Automated modernization of legacy scientific computing.
208
+
209
+ Expose native C++/C/Fortran code to Python as deployable microservices,
210
+ without rewriting the numerics and without hand-writing bindings.
211
+ """
212
+
213
+
214
+ @main.command()
215
+ def init() -> None:
216
+ """Scaffold the monorepo layout (design.md section 4)."""
217
+ for d in [SERVICES_DIR, "libraries", "tools", "infrastructure/docker", "infrastructure/kubernetes"]:
218
+ Path(d).mkdir(parents=True, exist_ok=True)
219
+ click.echo("Initialized nativegate monorepo layout.")
220
+
221
+
222
+ @main.command("create-service")
223
+ @click.argument("name")
224
+ @click.option("--language", type=click.Choice(["cpp", "fortran"]), default="cpp")
225
+ @click.option(
226
+ "--force",
227
+ is_flag=True,
228
+ help="Delete and recreate services/<name> if it already exists. Removes any native code already placed there.",
229
+ )
230
+ def create_service(name: str, language: str, force: bool) -> None:
231
+ """Scaffold a new services/<name> directory (design.md section 4)."""
232
+ service_dir = _scaffold_service(name, language, force)
233
+ click.echo(f"Created service '{name}' ({language}) at {service_dir}")
234
+ click.echo(f"Next: add native code under {service_dir / 'native'}, then run `ngate expose {service_dir / 'native'}`")
235
+
236
+
237
+ @main.command()
238
+ @click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path))
239
+ @click.option("--name", help="Service name. Defaults to SOURCE's filename without extension.")
240
+ @click.option(
241
+ "--force",
242
+ is_flag=True,
243
+ help="Delete and recreate services/<name> if it already exists.",
244
+ )
245
+ @click.option("--build/--no-build", default=False, help="Also compile the extension after generating.")
246
+ def quickstart(source: Path, name: str | None, force: bool, build: bool) -> None:
247
+ """One-shot: scaffold a service from a single C++/Fortran file, expose
248
+ everything found in it, and generate the Python package — no manual
249
+ nativegate.yaml editing.
250
+
251
+ Runs `create-service`, copies SOURCE into native/, auto-populates
252
+ `expose:` with every symbol found (all classes/functions for C++, every
253
+ routine name for Fortran), then `generate`. For anything beyond "expose
254
+ everything in one file" — partial exposure, multi-file services, huge
255
+ legacy Fortran templates where you only want one routine — use
256
+ create-service + nativegate.yaml + generate directly instead; see
257
+ the Fortran guide in docs/.
258
+ """
259
+ language = detect_language(source)
260
+ if language is None:
261
+ raise click.ClickException(
262
+ f"Could not detect a native language for {source} "
263
+ "(expected .hpp/.cpp for C++ or .f90/.f95/.f03 for Fortran)."
264
+ )
265
+
266
+ service_name = name or source.stem
267
+ steps = ["Scaffold service", "Copy native source", "Generate bindings & package"]
268
+ if build:
269
+ steps.append("Build wheel")
270
+
271
+ with StepTracker(steps) as tracker:
272
+ tracker.start("Scaffold service")
273
+ try:
274
+ service_dir = _scaffold_service(service_name, language, force)
275
+ except Exception:
276
+ tracker.fail("Scaffold service")
277
+ raise
278
+ tracker.log(f"services/{service_name}")
279
+ tracker.done("Scaffold service")
280
+
281
+ tracker.start("Copy native source")
282
+ native_copy = service_dir / "native" / source.name
283
+ # Copied byte-for-byte: a latin-1 legacy deck must not pick up
284
+ # U+FFFD replacement characters on its way into the service.
285
+ native_copy.write_bytes(source.read_bytes())
286
+ tracker.log(f"{source} -> {native_copy}")
287
+
288
+ config = _load_config(service_dir)
289
+ if language == "cpp":
290
+ # Empty expose: lists mean "expose everything found" for C++ — see ExposeConfig.is_exposed.
291
+ # A header with declarations only (no bodies) needs its matching
292
+ # .cpp — without it the extension still *builds* on macOS (undefined
293
+ # symbols are deferred to load time) but fails with a confusing
294
+ # dlopen error on first import. Pull in same-stem implementation
295
+ # files automatically, the same pairing calculator.hpp/calculator.cpp uses.
296
+ if source.suffix in (".hpp", ".hh", ".h"):
297
+ impl_files = find_implementation_files(source)
298
+ for impl_file in impl_files:
299
+ impl_copy = service_dir / "native" / impl_file.name
300
+ impl_copy.write_bytes(impl_file.read_bytes())
301
+ tracker.log(f"{impl_file} -> {impl_copy}")
302
+ if not impl_files:
303
+ tracker.log(
304
+ f"[yellow]WARNING[/yellow]: no implementation file found for {source.name} — "
305
+ "looked beside it and in any sibling src/ directory. If its "
306
+ "methods are declared but not defined inline, "
307
+ "`ngate build` will compile and then fail to import "
308
+ "(undefined symbol). Add the .cpp to "
309
+ f"{service_dir / 'native'} and re-run `ngate generate {service_name}`."
310
+ )
311
+ else:
312
+ routines = fortran_parser.list_routine_names(native_copy)
313
+ if not routines:
314
+ tracker.fail("Copy native source")
315
+ raise click.ClickException(f"No Fortran function/subroutine declarations found in {source}.")
316
+ config.expose.functions = routines
317
+ config.save(service_dir)
318
+ tracker.log(f"Exposing Fortran routines: {', '.join(routines)}")
319
+ tracker.done("Copy native source")
320
+
321
+ tracker.start("Generate bindings & package")
322
+ try:
323
+ _generate_service(service_dir, config)
324
+ except Exception:
325
+ tracker.fail("Generate bindings & package")
326
+ raise
327
+ tracker.log(f"services/{service_name}/python/{service_name}")
328
+ tracker.done("Generate bindings & package")
329
+
330
+ if build:
331
+ tracker.start("Build wheel")
332
+ try:
333
+ _run([sys.executable, "-m", "pip", "wheel", ".", "-w", "dist"], cwd=service_dir, log=tracker.log)
334
+ except SystemExit:
335
+ tracker.fail("Build wheel")
336
+ raise
337
+ tracker.log(f"services/{service_name}/dist/")
338
+ tracker.done("Build wheel")
339
+
340
+ click.echo(f"\nDone — services/{service_name} is ready.")
341
+
342
+
343
+ @main.command()
344
+ @click.argument("path", type=click.Path(exists=True, path_type=Path))
345
+ def detect(path: Path) -> None:
346
+ """Detect the native language of a file or directory."""
347
+ if path.is_file():
348
+ lang = detect_language(path)
349
+ click.echo(f"{path}: {lang or 'unknown'}")
350
+ return
351
+
352
+ for lang in ("cpp", "fortran"):
353
+ sources = find_native_sources(path, lang)
354
+ if sources:
355
+ click.echo(f"{lang}:")
356
+ for src in sources:
357
+ click.echo(f" {src}")
358
+
359
+
360
+ @main.command()
361
+ @click.argument("path", type=click.Path(exists=True, file_okay=False, path_type=Path))
362
+ @click.option(
363
+ "--parser",
364
+ type=click.Choice(list(cpp_parser.BACKENDS)),
365
+ default="auto",
366
+ help="C++ only: 'clang' for the AST parser, 'regex' for the fallback reader.",
367
+ )
368
+ @click.option(
369
+ "--include",
370
+ "includes",
371
+ multiple=True,
372
+ help="C++ only: extra include directory for the AST parse (repeatable).",
373
+ )
374
+ @click.option("--std", default="c++17", show_default=True, help="C++ only: language standard.")
375
+ @click.option("--all", "show_all", is_flag=True, help="Show every file, not just the top 15.")
376
+ def suggest(path: Path, parser: str, includes: tuple[str, ...], show_all: bool, std: str) -> None:
377
+ """Rank the native sources under PATH by how cleanly they would bind.
378
+
379
+ Answers "which file do I point nativegate at first?" for a codebase you
380
+ did not write. Parses every header it finds and sorts by how much binds
381
+ versus how much is skipped, preferring self-contained files — a good
382
+ first service is one with no dependencies on the rest of the tree, not
383
+ necessarily the biggest one.
384
+ """
385
+ console = Console()
386
+ if parser == "auto":
387
+ console.print(f"[dim]parser: {cpp_parser.backend_description(parser)}[/dim]")
388
+
389
+ with console.status(f"Parsing native sources under {path}..."):
390
+ candidates = analyse_tree(
391
+ path, backend=parser, options=ClangOptions(std=std, include_paths=tuple(includes))
392
+ )
393
+
394
+ if not candidates:
395
+ raise click.ClickException(f"No C++ or Fortran sources found under {path}.")
396
+
397
+ marks = {
398
+ READY: "[green]✔[/green]",
399
+ WORKABLE: "[yellow]~[/yellow]",
400
+ POOR: "[red]✘[/red]",
401
+ }
402
+ table = Table(box=box.SIMPLE, header_style="bold")
403
+ table.add_column("")
404
+ table.add_column("file", no_wrap=True)
405
+ table.add_column("binds")
406
+ table.add_column("skipped", justify="right")
407
+ table.add_column("notes", style="dim", overflow="fold")
408
+
409
+ shown = candidates if show_all else candidates[:15]
410
+ for c in shown:
411
+ # c.path is already relative to the cwd (find_native_sources builds it
412
+ # from the PATH argument as given), same as the "Start with" command
413
+ # below — stripping the root here would print a path missing the
414
+ # root it was found under, not one runnable as-is.
415
+ table.add_row(
416
+ marks[c.verdict],
417
+ str(c.path),
418
+ c.binds_summary(),
419
+ str(c.skipped) if c.skipped else "",
420
+ c.notes(),
421
+ )
422
+ console.print(table)
423
+ if len(candidates) > len(shown):
424
+ console.print(f"[dim]... and {len(candidates) - len(shown)} more (--all to show).[/dim]")
425
+
426
+ best = candidates[0]
427
+ if best.verdict == POOR:
428
+ console.print(
429
+ "[yellow]Nothing here binds cleanly.[/yellow] Check the skipped reasons with "
430
+ f"`ngate inspect <file>` — and if the parser line above says 'regex reader', "
431
+ 'install the AST parser first: pip install "nativegate[clang]".'
432
+ )
433
+ return
434
+
435
+ console.print(f"\nStart with [bold]{best.path}[/bold]:\n")
436
+ console.print(f" ngate quickstart {best.path} --name {best.path.stem.lower()} --build")
437
+
438
+
439
+ @main.command()
440
+ @click.argument("path", type=click.Path(exists=True, path_type=Path))
441
+ @click.option(
442
+ "--function",
443
+ "functions",
444
+ multiple=True,
445
+ help="Fortran only: name of a function/subroutine to extract (repeatable). Required for Fortran.",
446
+ )
447
+ @click.option(
448
+ "--parser",
449
+ type=click.Choice(list(cpp_parser.BACKENDS)),
450
+ default="auto",
451
+ help="C++ only: 'clang' for the AST parser, 'regex' for the fallback reader.",
452
+ )
453
+ @click.option(
454
+ "--include",
455
+ "includes",
456
+ multiple=True,
457
+ help="C++ only: extra include directory for the AST parse (repeatable).",
458
+ )
459
+ @click.option(
460
+ "--std",
461
+ default="c++17",
462
+ show_default=True,
463
+ help="C++ only: language standard for the AST parse.",
464
+ )
465
+ def inspect(
466
+ path: Path,
467
+ functions: tuple[str, ...],
468
+ parser: str,
469
+ includes: tuple[str, ...],
470
+ std: str,
471
+ ) -> None:
472
+ """Parse a native source file and print the resulting IR."""
473
+ lang = detect_language(path)
474
+ if lang == "cpp":
475
+ try:
476
+ click.echo(f"parser: {cpp_parser.backend_description(parser)}")
477
+ module = cpp_parser.parse_header(
478
+ path,
479
+ ExposeConfig(),
480
+ backend=parser,
481
+ options=ClangOptions(std=std, include_paths=tuple(includes)),
482
+ )
483
+ except cpp_parser.ParserUnavailable as exc:
484
+ raise click.ClickException(str(exc)) from exc
485
+ elif lang == "fortran":
486
+ if not functions:
487
+ raise click.ClickException(
488
+ "Fortran sources need at least one --function <name> to target "
489
+ "(large legacy files are parsed one routine at a time)."
490
+ )
491
+ module = fortran_parser.parse_source(path, ExposeConfig(functions=list(functions)))
492
+ else:
493
+ raise click.ClickException(f"Unrecognized native source: {path} (detected language: {lang}).")
494
+
495
+ click.echo(f"module: {module.name} ({module.language})")
496
+ for cls in module.classes:
497
+ click.echo(f" class {cls.name}")
498
+ for m in cls.methods:
499
+ params = ", ".join(f"{p.name}: {p.type}" for p in m.parameters)
500
+ click.echo(f" {m.name}({params}) -> {m.returns}")
501
+ for struct in module.structs:
502
+ click.echo(f" struct {struct.name}")
503
+ for f in struct.fields:
504
+ click.echo(f" {f.name}: {f.type}")
505
+ for fn in module.functions:
506
+ params = ", ".join(f"{p.name}: {p.type}" for p in fn.parameters)
507
+ click.echo(f" function {fn.name}({params}) -> {fn.returns}")
508
+ _report_skipped(path, module)
509
+ if module.is_empty():
510
+ click.echo(" (nothing exposed — add [[nativegate::expose]] or list symbols in nativegate.yaml)")
511
+
512
+
513
+ @main.command()
514
+ @click.argument("path", type=click.Path(exists=True, path_type=Path))
515
+ def expose(path: Path) -> None:
516
+ """Mark a native source as exposed and run codegen for its service.
517
+
518
+ PATH may be a header inside services/<name>/native/, or that native/
519
+ directory itself. Equivalent to editing nativegate.yaml + `generate`.
520
+ """
521
+ service_dir = _resolve_service_dir(path)
522
+ config = _load_config(service_dir)
523
+ _generate_service(service_dir, config)
524
+ click.echo(f"Exposed native API from {path} for service '{config.name}'.")
525
+
526
+
527
+ @main.command()
528
+ @click.argument("name")
529
+ def generate(name: str) -> None:
530
+ """Re-run binding/package/test generation for services/<name>."""
531
+ service_dir = _service_dir(name)
532
+ config = _load_config(service_dir)
533
+ _generate_service(service_dir, config)
534
+ click.echo(f"Generated bindings, CMake, Python package, and tests for '{name}'.")
535
+
536
+
537
+ @main.command()
538
+ @click.argument("name")
539
+ def build(name: str) -> None:
540
+ """Build the service's Python wheel (pip wheel .)."""
541
+ service_dir = _service_dir(name)
542
+ _run([sys.executable, "-m", "pip", "wheel", ".", "-w", "dist"], cwd=service_dir)
543
+
544
+
545
+ @main.group()
546
+ def golden() -> None:
547
+ """Numerical regression: record what the bindings return, then check it.
548
+
549
+ "It builds and imports" is not the acceptance criterion for re-hosting
550
+ decades-old engineering code — "the answers did not change" is. `record`
551
+ captures every bound entry point's output for a fixed, reproducible set of
552
+ inputs into services/<name>/golden.json; `verify` replays it. The
553
+ generated tests/test_golden.py does the same thing inside the service's
554
+ own suite, so CI catches drift without any extra wiring.
555
+ """
556
+
557
+
558
+ def _import_service_package(name: str):
559
+ """Import the service's built package, or explain how to get one.
560
+
561
+ Recording runs against the *compiled* extension, not the source: the whole
562
+ point is to catch a compiler, flag or code change that moves a number.
563
+ """
564
+ import importlib
565
+
566
+ try:
567
+ return importlib.import_module(name)
568
+ except ImportError as exc:
569
+ raise click.ClickException(
570
+ f"Could not import the '{name}' package ({exc}). Golden values are "
571
+ "recorded against the built extension, so build and install the "
572
+ f"service first:\n ngate build {name}\n"
573
+ f" pip install services/{name}/dist/*.whl"
574
+ ) from exc
575
+
576
+
577
+ def _require_installed_package(name: str) -> None:
578
+ """Fail with the build instructions rather than uvicorn's import traceback."""
579
+ import importlib.util
580
+
581
+ if importlib.util.find_spec(name) is None:
582
+ raise click.ClickException(
583
+ f"The '{name}' package is not importable. `serve` runs the built "
584
+ f"extension, so build and install the service first:\n"
585
+ f" ngate build {name}\n"
586
+ f" pip install services/{name}/dist/*.whl"
587
+ )
588
+
589
+
590
+ def _load_service_ir(service_dir: Path, name: str) -> ModuleIR:
591
+ ir_path = service_dir / IR_PATH
592
+ if not ir_path.exists():
593
+ raise click.ClickException(
594
+ f"No {ir_path} — run `ngate generate {name}` first."
595
+ )
596
+ try:
597
+ return module_from_dict(json.loads(ir_path.read_text()))
598
+ except IRSchemaError as exc:
599
+ # A version this nativegate cannot honestly read has to stop the command,
600
+ # not surface as a traceback from inside `golden record`.
601
+ raise click.ClickException(f"{ir_path}: {exc}") from exc
602
+
603
+
604
+ @golden.command("record")
605
+ @click.argument("name")
606
+ @click.option("--rtol", default=golden_lib.DEFAULT_RTOL, show_default=True, help="Relative tolerance stored in the golden file.")
607
+ @click.option("--atol", default=golden_lib.DEFAULT_ATOL, show_default=True, help="Absolute tolerance stored in the golden file.")
608
+ @click.option("--force", is_flag=True, help="Overwrite an existing golden file whose values differ.")
609
+ def golden_record(name: str, rtol: float, atol: float, force: bool) -> None:
610
+ """Record golden values for services/<name> (needs the built package installed)."""
611
+ service_dir = _service_dir(name)
612
+ module = _load_service_ir(service_dir, name)
613
+ package = _import_service_package(name)
614
+
615
+ path = service_dir / golden_lib.GOLDEN_FILENAME
616
+ existing = golden_lib.read(path) if path.exists() else None
617
+
618
+ # Hand-edited inputs are kept: an engineer who replaced the generated
619
+ # `1.0` with a real reservoir pressure should not lose it on the next
620
+ # re-record.
621
+ entries, skips = golden_lib.run(module, package, existing)
622
+
623
+ # Hash the sources this service actually compiles. Left to its default,
624
+ # `provenance()` records an empty `sources` map, which the generated
625
+ # golden test rejects — so a service built once could never build again,
626
+ # and the remedy the failure names (re-record) produced the same empty
627
+ # map. Resolved through oracle's helper so `oracle_check` recomputes an
628
+ # identical set rather than reporting a phantom source change.
629
+ sources_dir, source_names = oracle_lib.compiled_sources(service_dir, module.language)
630
+ document = golden_lib.build_document(
631
+ module,
632
+ entries,
633
+ skips,
634
+ rtol=rtol,
635
+ atol=atol,
636
+ environment=golden_lib.provenance(
637
+ sources=[sources_dir / n for n in source_names]
638
+ ),
639
+ )
640
+
641
+ if existing is not None and not force:
642
+ results = {key: entry["result"] for key, entry in entries.items()}
643
+ differences = golden_lib.compare(existing, results)
644
+ if differences:
645
+ # Re-recording silently is how a regression gets committed as the
646
+ # new truth. Make the operator say they meant it.
647
+ click.echo(f"{len(differences)} value(s) differ from the recorded golden file:")
648
+ for line in differences[:20]:
649
+ click.echo(f" - {line}")
650
+ raise click.ClickException(
651
+ "Refusing to overwrite. Investigate first; re-record with --force "
652
+ "once you are satisfied the change is intended."
653
+ )
654
+
655
+ golden_lib.write(path, document)
656
+ recorded, skipped = golden_lib.coverage(document)
657
+ click.echo(f"Recorded {recorded} entry point(s) to {path}.")
658
+ if skipped:
659
+ click.echo(f"{skipped} entry point(s) not covered:")
660
+ for key, reason in document["skipped"].items():
661
+ click.echo(f" - {key}: {reason}")
662
+
663
+
664
+ @golden.command("verify")
665
+ @click.argument("name")
666
+ def golden_verify(name: str) -> None:
667
+ """Replay services/<name>'s golden file against the installed package."""
668
+ _golden_verify(name)
669
+
670
+
671
+ def _golden_verify(name: str) -> None:
672
+ service_dir = _service_dir(name)
673
+ path = service_dir / golden_lib.GOLDEN_FILENAME
674
+ if not path.exists():
675
+ raise click.ClickException(
676
+ f"No {path} — record one first with `ngate golden record {name}`."
677
+ )
678
+
679
+ document = golden_lib.read(path)
680
+ package = _import_service_package(name)
681
+ results, errors = golden_lib.replay(document, package)
682
+ differences = golden_lib.compare(document, results, errors)
683
+
684
+ if differences:
685
+ click.echo(f"{len(differences)} numerical difference(s):")
686
+ for line in differences:
687
+ click.echo(f" - {line}")
688
+ raise click.ClickException(
689
+ "The bindings no longer return the recorded values. If the change is "
690
+ f"intended, re-record with `ngate golden record {name} --force`."
691
+ )
692
+
693
+ recorded, skipped = golden_lib.coverage(document)
694
+ click.echo(f"{recorded} entry point(s) unchanged ({skipped} not covered).")
695
+
696
+
697
+ @golden.command("show")
698
+ @click.argument("name")
699
+ def golden_show(name: str) -> None:
700
+ """Print what the golden file covers, and what it does not."""
701
+ service_dir = _service_dir(name)
702
+ path = service_dir / golden_lib.GOLDEN_FILENAME
703
+ if not path.exists():
704
+ raise click.ClickException(f"No {path} — record one first.")
705
+
706
+ document = golden_lib.read(path)
707
+ recorded, skipped = golden_lib.coverage(document)
708
+ tolerance = document.get("tolerance") or {}
709
+ click.echo(
710
+ f"{path}: {recorded} recorded, {skipped} not covered "
711
+ f"(rtol={tolerance.get('rtol')}, atol={tolerance.get('atol')})"
712
+ )
713
+ for key, entry in (document.get("entries") or {}).items():
714
+ click.echo(f" {key}({', '.join(repr(a) for a in entry['arguments'])}) -> {entry['result']!r}")
715
+ for key, reason in (document.get("skipped") or {}).items():
716
+ click.echo(f" [skipped] {key}: {reason}")
717
+
718
+
719
+ def _package_namespace_for(module: ModuleIR, package):
720
+ """The object `golden.invoke()`/T10/T11's runners call attributes on --
721
+ same rule as `oracle._package_namespace`: f2py nests everything under
722
+ the enclosing `module X` block, if there is one."""
723
+ if module.fortran_module:
724
+ return getattr(package, module.fortran_module)
725
+ return package
726
+
727
+
728
+ def _subprocess_target_for(name: str, module: ModuleIR) -> si_lib.SubprocessTarget:
729
+ """How T10's fresh-process worker re-imports the *installed* package --
730
+ no `sys_path` entry needed (unlike oracle's freshly-built extension,
731
+ which lives in a temp build dir), since `_import_service_package`
732
+ already required this be importable."""
733
+ attr_path = (module.fortran_module,) if module.fortran_module else ()
734
+ return si_lib.SubprocessTarget(module_name=name, sys_path=(), attr_path=attr_path)
735
+
736
+
737
+ def _invariants_verify(name: str) -> None:
738
+ """Shared by `invariants verify` and the `verify` aggregate (spec section
739
+ 2.6: `ngate verify` runs invariants "if `invariants.json` exists" --
740
+ read here as "if `state:`/`invariants:`/`ranges:` are declared", since
741
+ those declarations are the source of truth and the file is a result."""
742
+ service_dir = _service_dir(name)
743
+ module = _load_service_ir(service_dir, name)
744
+ try:
745
+ config = ServiceConfig.load(service_dir)
746
+ except FileNotFoundError as exc:
747
+ raise click.ClickException(str(exc)) from exc
748
+ try:
749
+ config.verification.validate_against_ir(module)
750
+ except ConfigError as exc:
751
+ raise click.ClickException(str(exc)) from exc
752
+
753
+ golden_path = service_dir / golden_lib.GOLDEN_FILENAME
754
+ if not golden_path.exists():
755
+ raise click.ClickException(
756
+ f"No {golden_path} — record one first with `ngate golden record {name}` "
757
+ "(declared invariants are checked against golden.json's recorded arguments)."
758
+ )
759
+ document = golden_lib.read(golden_path)
760
+ package = _import_service_package(name)
761
+ target = _subprocess_target_for(name, module)
762
+
763
+ try:
764
+ result = invariants_lib.verify_invariants(
765
+ module, document, config.verification, _package_namespace_for(module, package), target
766
+ )
767
+ except invariants_lib.EmptyCheckedError as exc:
768
+ raise click.ClickException(str(exc)) from exc
769
+
770
+ path = service_dir / invariants_lib.INVARIANTS_FILENAME
771
+ invariants_lib.write(path, result.document)
772
+
773
+ checked, uncovered = invariants_lib.coverage(result.document)
774
+ click.echo(f"{checked} function(s) checked, {uncovered} uncovered ({path}):")
775
+ for key, entry in result.document["checked"].items():
776
+ click.echo(
777
+ f" {key}: {entry['status']} ({len(entry['properties'])} propert(y/ies), "
778
+ f"{entry['points']} point(s))"
779
+ )
780
+ for key, reason in result.document["uncovered"].items():
781
+ click.echo(f" [uncovered] {key}: {reason}")
782
+
783
+ if not result.passed:
784
+ click.echo(f"{len(result.failure_messages)} invariant failure(s):")
785
+ for message in result.failure_messages:
786
+ click.echo(f" - {message}")
787
+ raise click.ClickException(f"Declared/structural invariants failed for '{name}'.")
788
+
789
+
790
+ @main.group()
791
+ def invariants() -> None:
792
+ """Layer 3: are the declared properties true over the swept lattice?
793
+
794
+ `verify` runs T10's structural properties (`finite`, `total`,
795
+ `no_error_flag`, `idempotent`, `order_independent`) and T11's declared
796
+ properties (`bounds`, `monotone`, `sum_to_one`) over golden.json's
797
+ recorded entries, swept per nativegate.yaml's `ranges:`, and writes
798
+ services/<name>/invariants.json (design-verification-layers.md section
799
+ 3.7).
800
+ """
801
+
802
+
803
+ @invariants.command("verify")
804
+ @click.argument("name")
805
+ def invariants_verify(name: str) -> None:
806
+ """Check services/<name>'s declared invariants and write invariants.json."""
807
+ _invariants_verify(name)
808
+
809
+
810
+ def _toolchain_present() -> bool:
811
+ """Whether *a* compiler this repo could build a driver with is on PATH --
812
+ `oracle check` needs one (design-verification-layers.md section 5: "needs
813
+ a compiler"); `golden`/`invariants` do not."""
814
+ return any(shutil.which(tool) for tool in ("gfortran", "clang++", "g++"))
815
+
816
+
817
+ @main.command()
818
+ @click.argument("name")
819
+ def verify(name: str) -> None:
820
+ """Run every verification layer for services/<name>, reporting each
821
+ separately.
822
+
823
+ Order (design-verification-layers.md section 2.6 / section 5's CI
824
+ ordering, restated verbatim): **oracle check first** — a faithful
825
+ binding is a precondition for the other two layers meaning anything —
826
+ **then golden, then invariants**. `oracle check` needs a compiler and is
827
+ skipped (visibly, not silently) if none is on PATH; `golden verify` and
828
+ `invariants verify` need only the installed wheel. A failure in one
829
+ layer is reported by name and does not stop the others from running, so
830
+ one layer's failure can never mask another's.
831
+ """
832
+ service_dir = _service_dir(name)
833
+ failed_layers: list[str] = []
834
+
835
+ # 1. oracle check --------------------------------------------------------
836
+ if not _toolchain_present():
837
+ click.echo("oracle: skipped, no toolchain")
838
+ else:
839
+ try:
840
+ report = oracle_lib.oracle_check(name, service_dir=service_dir)
841
+ except oracle_lib.OracleError as exc:
842
+ click.echo(f"oracle: FAILED — {exc}")
843
+ failed_layers.append("oracle")
844
+ else:
845
+ for line in report.failures:
846
+ click.echo(f" - {line}")
847
+ if report.passed:
848
+ click.echo(
849
+ f"oracle: passed ({report.covered} covered, {len(report.skipped)} skipped)"
850
+ )
851
+ else:
852
+ click.echo(f"oracle: FAILED ({len(report.failures)} failure(s))")
853
+ failed_layers.append("oracle")
854
+
855
+ # 2. golden ---------------------------------------------------------------
856
+ try:
857
+ _golden_verify(name)
858
+ except click.ClickException as exc:
859
+ click.echo(f"golden: FAILED — {exc.message}")
860
+ failed_layers.append("golden")
861
+ else:
862
+ click.echo("golden: passed")
863
+
864
+ # 3. invariants -------------------------------------------------------------
865
+ config = None
866
+ config_invalid = False
867
+ try:
868
+ config = ServiceConfig.load(service_dir)
869
+ except FileNotFoundError:
870
+ pass
871
+ except ConfigError as exc:
872
+ # A malformed nativegate.yaml must be reported as this layer's
873
+ # failure, same as every other invariants error below — never let it
874
+ # propagate as a raw traceback that aborts golden/oracle's already
875
+ # -reported results too.
876
+ click.echo(f"invariants: FAILED — {exc}")
877
+ failed_layers.append("invariants")
878
+ config_invalid = True
879
+
880
+ if config_invalid:
881
+ pass
882
+ elif config is None or config.verification.is_empty:
883
+ click.echo("invariants: skipped, no state/invariants/ranges declared")
884
+ else:
885
+ try:
886
+ _invariants_verify(name)
887
+ except click.ClickException as exc:
888
+ click.echo(f"invariants: FAILED — {exc.message}")
889
+ failed_layers.append("invariants")
890
+ else:
891
+ click.echo("invariants: passed")
892
+
893
+ if failed_layers:
894
+ raise click.ClickException(
895
+ f"verification failed for '{name}': {', '.join(failed_layers)} layer(s) failed."
896
+ )
897
+
898
+
899
+ @main.group()
900
+ def oracle() -> None:
901
+ """Layer 2: is the binding faithful to the legacy binary, not just unchanged?
902
+
903
+ `check` generates a native driver from the entries recorded in
904
+ golden.json (never from a fresh sample plan — see design-verification-
905
+ layers.md section 2.2), builds and runs it, executes the same calls
906
+ through the Python binding in the same build, and compares every
907
+ observable value bitwise. It regenerates and recompiles the driver every
908
+ time — there is no committed file to go stale, and none is needed to
909
+ fail a build (design-verification-layers.md section 2.6).
910
+ """
911
+
912
+
913
+ @oracle.command("check")
914
+ @click.argument("name")
915
+ def oracle_check(name: str) -> None:
916
+ """Generate, build and run the oracle driver for services/<name>, and
917
+ compare it bitwise against the Python binding, in this build."""
918
+ service_dir = _service_dir(name)
919
+ try:
920
+ report = oracle_lib.oracle_check(name, service_dir=service_dir)
921
+ except oracle_lib.OracleError as exc:
922
+ raise click.ClickException(str(exc)) from exc
923
+
924
+ if report.failures:
925
+ click.echo(f"{len(report.failures)} oracle failure(s):")
926
+ for line in report.failures:
927
+ click.echo(f" - {line}")
928
+
929
+ click.echo(
930
+ f"{report.covered} covered, {len(report.skipped)} skipped"
931
+ + (
932
+ " (" + ", ".join(f"{k}: {v}" for k, v in report.skipped.items()) + ")"
933
+ if report.skipped
934
+ else ""
935
+ )
936
+ )
937
+
938
+ if report.historical_diff_refused:
939
+ click.echo(report.historical_diff_refused)
940
+ elif report.historical_diff:
941
+ click.echo(f"{len(report.historical_diff)} bit(s) differ from the recorded oracle.json:")
942
+ for line in report.historical_diff:
943
+ click.echo(f" - {line}")
944
+
945
+ if not report.passed:
946
+ raise click.ClickException(
947
+ f"The oracle disagrees with the Python binding for '{name}' — "
948
+ "the binding is not faithful to the native code it was "
949
+ "generated from, or the check covered nothing."
950
+ )
951
+
952
+
953
+ @oracle.command("record")
954
+ @click.argument("name")
955
+ def oracle_record(name: str) -> None:
956
+ """Run a full oracle check for services/<name> and, on pass, write
957
+ oracle.json — provenance, not the gate (design-verification-layers.md
958
+ section 2.6). A future `check` in the same build image additionally
959
+ diffs against it bitwise; anywhere else, the CLI refuses that historical
960
+ comparison rather than falling back to a tolerance."""
961
+ service_dir = _service_dir(name)
962
+ try:
963
+ document = oracle_lib.oracle_record(name, service_dir=service_dir)
964
+ except oracle_lib.OracleError as exc:
965
+ raise click.ClickException(str(exc)) from exc
966
+
967
+ path = service_dir / oracle_lib.ORACLE_FILENAME
968
+ covered, skipped = len(document["entries"]), len(document["skipped"])
969
+ click.echo(f"Recorded {covered} entry point(s) to {path}.")
970
+ if skipped:
971
+ click.echo(f"{skipped} entry point(s) not covered:")
972
+ for key, reason in document["skipped"].items():
973
+ click.echo(f" - {key}: {reason}")
974
+
975
+
976
+ @oracle.command("show")
977
+ @click.argument("name")
978
+ def oracle_show(name: str) -> None:
979
+ """Print services/<name>'s golden.json entries, their expected wire
980
+ slots, and the skips — without building, compiling, or running anything."""
981
+ service_dir = _service_dir(name)
982
+ try:
983
+ report = oracle_lib.oracle_show(name, service_dir=service_dir)
984
+ except oracle_lib.OracleError as exc:
985
+ raise click.ClickException(str(exc)) from exc
986
+
987
+ click.echo(f"{len(report.entries)} entries, {len(report.skipped)} skipped:")
988
+ for entry in report.entries:
989
+ args = ", ".join(repr(a) for a in entry.arguments)
990
+ if entry.slots is None:
991
+ click.echo(f" {entry.key}({args}) -> [no matching function in the IR]")
992
+ else:
993
+ click.echo(f" {entry.key}({args}) -> {', '.join(entry.slots) or '(no observable slots)'}")
994
+ for key, reason in report.skipped.items():
995
+ click.echo(f" [skipped] {key}: {reason}")
996
+
997
+
998
+ @main.command()
999
+ @click.argument("name")
1000
+ @click.option("--host", default="127.0.0.1", show_default=True, help="Interface to bind.")
1001
+ @click.option("--port", default=8000, show_default=True, help="Port to bind.")
1002
+ @click.option("--reload", is_flag=True, help="Restart on source changes (development only).")
1003
+ def serve(name: str, host: str, port: int, reload: bool) -> None:
1004
+ """Run services/<name>'s FastAPI app with uvicorn.
1005
+
1006
+ A convenience wrapper, not a deployment: it runs a single uvicorn process
1007
+ against the *installed* package. The generated Dockerfile runs gunicorn
1008
+ with uvicorn workers, and that is what should serve real traffic. The
1009
+ default bind is loopback, because a generated service is not hardened for
1010
+ an untrusted network (see docs/production-readiness.md).
1011
+ """
1012
+ _service_dir(name)
1013
+ _require_installed_package(name)
1014
+ import importlib.util
1015
+
1016
+ if importlib.util.find_spec("uvicorn") is None:
1017
+ raise click.ClickException(
1018
+ "uvicorn is not installed in this interpreter. Install the "
1019
+ f"service's own dependencies:\n"
1020
+ f" pip install services/{name}/dist/*.whl"
1021
+ )
1022
+
1023
+ cmd = [sys.executable, "-m", "uvicorn", f"{name}.service:app",
1024
+ "--host", host, "--port", str(port)]
1025
+ if reload:
1026
+ cmd.append("--reload")
1027
+ _run(cmd, cwd=Path.cwd())
1028
+
1029
+
1030
+ @main.command()
1031
+ @click.argument("name")
1032
+ def test(name: str) -> None:
1033
+ """Run the service's test suite."""
1034
+ service_dir = _service_dir(name)
1035
+ _run([sys.executable, "-m", "pytest", "tests/"], cwd=service_dir)
1036
+
1037
+
1038
+ @main.command()
1039
+ @click.argument("name")
1040
+ @click.option("--build/--no-build", default=False, help="Also run `docker build` after generating the Dockerfile.")
1041
+ def docker(name: str, build: bool) -> None:
1042
+ """Generate (and optionally build) the service's Dockerfile."""
1043
+ service_dir = _service_dir(name)
1044
+ config = _load_config(service_dir)
1045
+ # `libraries:` means two different things by language, and the Dockerfile
1046
+ # only has to care about one of them. C++ LINKS a CMake target that lives
1047
+ # outside the service, so the build context has to be the repo root. The
1048
+ # f2py path has no target to link: `generate` already copied those sources
1049
+ # into native/_expanded/, so everything the build needs is under the
1050
+ # service directory and the simpler context still works. Passing the
1051
+ # libraries through here would switch the context and emit a
1052
+ # `COPY libraries/...` for sources that are already vendored.
1053
+ libraries = [] if config.language == "fortran" else _validated_libraries(config)
1054
+ # Generated from what is actually there: a lock the user has not created
1055
+ # would produce a Dockerfile with a COPY that fails, and silently ignoring
1056
+ # one they HAVE created would quietly drop the pinning they asked for.
1057
+ locked = (service_dir / locking.LOCK_FILENAME).exists()
1058
+ dockerfile = docker_gen.generate_dockerfile(
1059
+ name, config.language, name, libraries=libraries, locked=locked
1060
+ )
1061
+ (service_dir / "Dockerfile").write_text(dockerfile)
1062
+ click.echo(f"Wrote {service_dir / 'Dockerfile'}")
1063
+ if locked:
1064
+ click.echo(" Dependencies install from requirements.lock (--require-hashes).")
1065
+ else:
1066
+ click.echo(
1067
+ f" NOTE: no {locking.LOCK_FILENAME} — pip will resolve dependencies at "
1068
+ f"build time, so two builds can differ. Run `ngate lock {name}`."
1069
+ )
1070
+
1071
+ if libraries:
1072
+ # Shared libraries live outside the service dir, so the build context
1073
+ # must be the repo root for COPY to reach them.
1074
+ click.echo(f"Build context: repo root (links libraries: {', '.join(libraries)})")
1075
+ if build:
1076
+ _run(
1077
+ ["docker", "build", "-f", str(service_dir / "Dockerfile"), "-t", f"{name}:latest", "."],
1078
+ cwd=Path("."),
1079
+ )
1080
+ elif build:
1081
+ _run(["docker", "build", "-t", f"{name}:latest", "."], cwd=service_dir)
1082
+
1083
+
1084
+ @main.command("lock")
1085
+ @click.argument("name")
1086
+ def lock(name: str) -> None:
1087
+ """Pin this service's Python dependencies by version and SHA-256.
1088
+
1089
+ Writes requirements.lock, which the generated Dockerfile installs with
1090
+ --require-hashes. Without it, `docker build` resolves fastapi/uvicorn/numpy
1091
+ from PyPI every time, so an image rebuilt a week later ships different code
1092
+ with nothing recording the change.
1093
+
1094
+ Resolution targets the IMAGE — Python 3.12 on Linux, both architectures —
1095
+ not the machine running this command.
1096
+ """
1097
+ service_dir = _service_dir(name)
1098
+ config = _load_config(service_dir)
1099
+
1100
+ requirements = list(
1101
+ docker_gen._LANGUAGE_RUNTIME_PYTHON_DEPS.get(
1102
+ config.language, docker_gen._LANGUAGE_RUNTIME_PYTHON_DEPS["cpp"]
1103
+ )
1104
+ )
1105
+ click.echo(f"Resolving {len(requirements)} direct dependencies for the image...")
1106
+ try:
1107
+ target = locking.write_lock(service_dir, name, requirements)
1108
+ except locking.LockError as exc:
1109
+ raise click.ClickException(str(exc)) from exc
1110
+
1111
+ pinned = sum(1 for line in target.read_text().splitlines() if "==" in line)
1112
+ click.echo(f"Wrote {target} ({pinned} packages pinned by hash)")
1113
+ click.echo(f" Re-run `ngate docker {name}` so the Dockerfile installs from it.")
1114
+
1115
+
1116
+ @main.command("k8s")
1117
+ @click.argument("name")
1118
+ @click.option("--image", default=None, help="Image reference to deploy. Defaults to <name>:latest.")
1119
+ @click.option("--replicas", default=2, show_default=True, help="Initial replica count.")
1120
+ @click.option(
1121
+ "--output",
1122
+ "output",
1123
+ default=None,
1124
+ help="Where to write the manifests. Defaults to infrastructure/kubernetes/<name>.yaml.",
1125
+ )
1126
+ def k8s(name: str, image: str | None, replicas: int, output: str | None) -> None:
1127
+ """Generate Kubernetes manifests for services/<name>.
1128
+
1129
+ Replicas, resources and the image are starting points. The security
1130
+ context and the probe wiring are not — see generators/k8s_gen.py.
1131
+ """
1132
+ service_dir = _service_dir(name)
1133
+ config = _load_config(service_dir)
1134
+
1135
+ manifests = k8s_gen.generate_k8s_manifests(
1136
+ name,
1137
+ config.language,
1138
+ image,
1139
+ auth=config.api.auth,
1140
+ replicas=replicas,
1141
+ )
1142
+ target = Path(output) if output else Path("infrastructure/kubernetes") / f"{name}.yaml"
1143
+ target.parent.mkdir(parents=True, exist_ok=True)
1144
+ target.write_text(manifests)
1145
+
1146
+ click.echo(f"Wrote {target}")
1147
+ click.echo(f" readiness -> /readyz (drains on SIGTERM), liveness -> /healthz")
1148
+ if config.language == "fortran":
1149
+ click.echo(
1150
+ " WEB_CONCURRENCY=1 — COMMON blocks are per-process. Scale with "
1151
+ "replicas, not workers."
1152
+ )
1153
+ if config.api.auth == "api_key":
1154
+ click.echo(
1155
+ f" Create the Secret first, or the pod will refuse to start:\n"
1156
+ f" kubectl create secret generic {name}-api-keys "
1157
+ '--from-literal=keys="$(openssl rand -hex 32)"'
1158
+ )
1159
+
1160
+
1161
+ @main.command()
1162
+ @click.argument("name")
1163
+ @click.option(
1164
+ "--service",
1165
+ "services",
1166
+ multiple=True,
1167
+ required=True,
1168
+ help="Service to mount into the gateway (repeatable). Each is mounted under /<service-name>.",
1169
+ )
1170
+ def gateway(name: str, services: tuple[str, ...]) -> None:
1171
+ """Generate a composed gateway app that serves several services on one URL.
1172
+
1173
+ Each service keeps its own wheel, version, and build — the gateway just
1174
+ depends on them and mounts their routers under a path prefix. Use this
1175
+ when you want one deployable and one URL; use a real API gateway /
1176
+ Kubernetes Ingress in front of separate images instead when you need
1177
+ independent scaling. See docs/deployment-topologies.md.
1178
+ """
1179
+ for service_name in services:
1180
+ _service_dir(service_name) # fail early if any service doesn't exist
1181
+
1182
+ # The distribution name may contain hyphens ("platform-api"), but the
1183
+ # importable package must be a valid Python identifier so uvicorn's
1184
+ # "<module>.app:app" target resolves.
1185
+ package_name = name.replace("-", "_")
1186
+
1187
+ gateway_dir = Path("gateways") / name
1188
+ package_dir = gateway_dir / package_name
1189
+ package_dir.mkdir(parents=True, exist_ok=True)
1190
+
1191
+ (package_dir / "__init__.py").write_text("")
1192
+ # The gateway is its own FastAPI app, so it needs its own middleware —
1193
+ # a mounted router runs under THIS app, not under the service's. Its auth
1194
+ # mode is the strictest of the services it mounts: composing an
1195
+ # authenticated service into an open gateway would publish it unprotected.
1196
+ auths = {_load_config(_service_dir(s)).api.auth for s in services}
1197
+ gateway_auth = "api_key" if "api_key" in auths else "none"
1198
+ _write_python(
1199
+ package_dir / middleware_gen.MIDDLEWARE_FILENAME,
1200
+ middleware_gen.generate_middleware_py(package_name, auth=gateway_auth),
1201
+ )
1202
+ try:
1203
+ app_source = gateway_gen.generate_gateway_app(name, list(services), package_name)
1204
+ mcp_source = mcp_gen.generate_mcp_py(name, list(services))
1205
+ except ValueError as exc:
1206
+ raise click.ClickException(str(exc)) from exc
1207
+ # One MCP endpoint covering every mounted service, built from the same
1208
+ # prefixed routers the gateway app mounts.
1209
+ _write_python(package_dir / mcp_gen.MCP_FILENAME, mcp_source)
1210
+ _write_python(package_dir / "app.py", app_source)
1211
+ (gateway_dir / "pyproject.toml").write_text(
1212
+ gateway_gen.generate_gateway_pyproject(name, package_name, list(services))
1213
+ )
1214
+
1215
+ click.echo(f"Generated gateway '{name}' at {gateway_dir}")
1216
+ click.echo(f"Mounts: {', '.join(f'/{s}' for s in services)}")
1217
+ if gateway_auth == "api_key":
1218
+ click.echo(
1219
+ "Auth: api_key (inherited from a mounted service). Set "
1220
+ f"{middleware_gen.ENV_API_KEYS} or the gateway will refuse to start."
1221
+ )
1222
+ click.echo(f"Run with: uvicorn {package_name}.app:app")
1223
+
1224
+
1225
+ @main.command()
1226
+ @click.argument("name")
1227
+ def clean(name: str) -> None:
1228
+ """Remove build artifacts for services/<name>."""
1229
+ import shutil
1230
+
1231
+ service_dir = _service_dir(name)
1232
+ for pattern in ["dist", "build", "*.egg-info", "**/__pycache__", ".pytest_cache", "native/_expanded"]:
1233
+ for match in service_dir.glob(pattern):
1234
+ if match.is_dir():
1235
+ shutil.rmtree(match, ignore_errors=True)
1236
+ else:
1237
+ match.unlink(missing_ok=True)
1238
+ click.echo(f"Cleaned build artifacts for '{name}'.")
1239
+
1240
+
1241
+ def _scaffold_service(name: str, language: str, force: bool) -> Path:
1242
+ import shutil
1243
+
1244
+ service_dir = Path(SERVICES_DIR) / name
1245
+ if service_dir.exists():
1246
+ if not force:
1247
+ raise click.ClickException(
1248
+ f"{service_dir} already exists. Re-run with --force to delete and recreate it."
1249
+ )
1250
+ click.echo(f"--force: removing existing {service_dir}")
1251
+ shutil.rmtree(service_dir)
1252
+
1253
+ (service_dir / "native").mkdir(parents=True)
1254
+ (service_dir / "bindings" / "generated").mkdir(parents=True)
1255
+ (service_dir / "python" / name).mkdir(parents=True)
1256
+ (service_dir / "tests").mkdir(parents=True)
1257
+
1258
+ ServiceConfig(name=name, language=language).save(service_dir)
1259
+ (service_dir / "python" / name / "__init__.py").write_text(
1260
+ "# Run `ngate expose` then `ngate generate` to populate this package.\n"
1261
+ )
1262
+ (service_dir / "pyproject.toml").write_text(pyproject_gen.generate_pyproject(name, language))
1263
+ return service_dir
1264
+
1265
+
1266
+ def _resolve_service_dir(path: Path) -> Path:
1267
+ for parent in [path] + list(path.parents):
1268
+ if (parent / "nativegate.yaml").exists():
1269
+ return parent
1270
+ if parent.name == "native" and (parent.parent / "nativegate.yaml").exists():
1271
+ return parent.parent
1272
+ raise click.ClickException(
1273
+ f"Could not find a nativegate.yaml above {path}. Run `ngate create-service` first."
1274
+ )
1275
+
1276
+
1277
+ def _generate_service(service_dir: Path, config: ServiceConfig) -> None:
1278
+ if config.language == "cpp":
1279
+ _generate_cpp_service(service_dir, config)
1280
+ elif config.language == "fortran":
1281
+ _generate_fortran_service(service_dir, config)
1282
+ else:
1283
+ raise click.ClickException(f"Unsupported language '{config.language}'.")
1284
+
1285
+
1286
+ IR_PATH = Path(".nativegate") / "ir.json"
1287
+
1288
+
1289
+ def _restore_scaffold(service_dir: Path, config: ServiceConfig) -> None:
1290
+ """Re-create scaffold files that `create-service` wrote, if they are gone.
1291
+
1292
+ Only ever writes what is *missing*: pyproject.toml is generated, but it is
1293
+ also the file you legitimately hand-edit (a pinned dependency, a version
1294
+ bump), so regenerating over it would silently discard that. Without this,
1295
+ deleting one generated file left `generate` unable to restore it and
1296
+ `create-service --force` — which deletes native/ — as the only way back.
1297
+ """
1298
+ pyproject = service_dir / "pyproject.toml"
1299
+ if not pyproject.exists():
1300
+ pyproject.write_text(
1301
+ pyproject_gen.generate_pyproject(config.name, config.language)
1302
+ )
1303
+ click.echo(f"Restored missing {pyproject}")
1304
+
1305
+
1306
+ def _warn_if_numpy_is_undeclared(service_dir: Path, module) -> None:
1307
+ """Say so when a binding needs numpy and pyproject.toml does not declare it.
1308
+
1309
+ A raw `T*` is bound through pybind11/numpy.h, which needs numpy's headers
1310
+ to build and numpy to import. pyproject.toml is written once at scaffold
1311
+ time — before any header has been parsed — and is explicitly a file you may
1312
+ hand-edit, so `generate` will not rewrite it. Telling you exactly what to
1313
+ add beats either clobbering your edits or letting the build fail with a
1314
+ missing-header error three layers down in CMake.
1315
+ """
1316
+ if not cmake_gen.uses_numpy_buffers(module):
1317
+ return
1318
+ pyproject = service_dir / "pyproject.toml"
1319
+ if pyproject.exists() and '"numpy"' in pyproject.read_text():
1320
+ return
1321
+ click.echo(
1322
+ "WARNING: this service binds a raw pointer argument as a numpy buffer, "
1323
+ "which needs numpy at build and run time, but pyproject.toml does not "
1324
+ 'declare it. Add "numpy" to [build-system] requires and to '
1325
+ "[project] dependencies."
1326
+ )
1327
+
1328
+
1329
+ def _write_package(service_dir: Path, config: ServiceConfig, module) -> None:
1330
+ _validate_module(module)
1331
+ _restore_scaffold(service_dir, config)
1332
+ _warn_if_numpy_is_undeclared(service_dir, module)
1333
+
1334
+ # A machine-readable record of exactly what was bound. `golden record`
1335
+ # reads it instead of re-parsing: recording happens against an installed
1336
+ # wheel, on a machine that may have neither the headers nor libclang.
1337
+ ir_path = service_dir / IR_PATH
1338
+ ir_path.parent.mkdir(parents=True, exist_ok=True)
1339
+ ir_path.write_text(json.dumps(module_to_dict(module), indent=2) + "\n")
1340
+
1341
+ package_dir = service_dir / "python" / config.name
1342
+ package_dir.mkdir(parents=True, exist_ok=True)
1343
+ # Every generated .py is compiled before it is written (D3): an unparsable
1344
+ # file must fail the build here, never the container start.
1345
+ _write_python(
1346
+ package_dir / "__init__.py", python_pkg_gen.generate_init_py(module), module
1347
+ )
1348
+ _write_python(
1349
+ package_dir / "router.py",
1350
+ python_pkg_gen.generate_router_py(module, config.name),
1351
+ module,
1352
+ )
1353
+ _write_python(
1354
+ package_dir / middleware_gen.MIDDLEWARE_FILENAME,
1355
+ middleware_gen.generate_middleware_py(config.name, auth=config.api.auth),
1356
+ module,
1357
+ )
1358
+ # The MCP view of the same router, mounted by service.py at /mcp. Written
1359
+ # before service.py only for readability; neither imports at generate time.
1360
+ _write_python(
1361
+ package_dir / mcp_gen.MCP_FILENAME,
1362
+ mcp_gen.generate_mcp_py(config.name),
1363
+ module,
1364
+ )
1365
+ _write_python(
1366
+ package_dir / "service.py", python_pkg_gen.generate_service_py(config.name), module
1367
+ )
1368
+
1369
+ tests_dir = service_dir / "tests"
1370
+ tests_dir.mkdir(parents=True, exist_ok=True)
1371
+ _write_python(
1372
+ tests_dir / "test_mcp.py",
1373
+ mcp_gen.generate_mcp_smoke_test(config.name),
1374
+ module,
1375
+ )
1376
+ _write_python(
1377
+ tests_dir / "test_python_api.py",
1378
+ test_gen.generate_python_api_test(module, config.name),
1379
+ module,
1380
+ )
1381
+ # The regression test ships even before anything is recorded: it skips
1382
+ # with the command to record, which is how you find out the harness
1383
+ # exists. A golden file that nobody knows to create protects nothing.
1384
+ _write_python(
1385
+ tests_dir / "test_golden.py",
1386
+ golden_gen.generate_golden_test(
1387
+ config.name, config.name, golden_lib.GOLDEN_FILENAME
1388
+ ),
1389
+ module,
1390
+ )
1391
+
1392
+
1393
+ LIBRARIES_DIR = "libraries"
1394
+
1395
+
1396
+ def _validated_libraries(config: ServiceConfig) -> list[str]:
1397
+ """Check each declared shared library exists and is buildable before we
1398
+ emit an add_subdirectory() that would otherwise fail deep inside CMake
1399
+ with a much less obvious message."""
1400
+ for lib in config.libraries:
1401
+ lib_dir = Path(LIBRARIES_DIR) / lib
1402
+ if not lib_dir.is_dir():
1403
+ raise click.ClickException(
1404
+ f"Service '{config.name}' declares library '{lib}' but "
1405
+ f"{lib_dir} does not exist."
1406
+ )
1407
+ if not (lib_dir / "CMakeLists.txt").exists():
1408
+ raise click.ClickException(
1409
+ f"{lib_dir} has no CMakeLists.txt — a shared native library must "
1410
+ "define its own CMake target (e.g. add_library(common_cpp STATIC ...) "
1411
+ "with POSITION_INDEPENDENT_CODE ON)."
1412
+ )
1413
+ return list(config.libraries)
1414
+
1415
+
1416
+ def _fortran_library_inputs(config: ServiceConfig) -> tuple[list[Path], list[Path]]:
1417
+ """Fortran sources and INCLUDE directories from this service's `libraries:`.
1418
+
1419
+ THE GAP THIS CLOSES
1420
+
1421
+ `libraries:` was C++-only. The C++ path links a shared library through
1422
+ CMake `add_subdirectory` + `target_link_libraries`; f2py has no equivalent,
1423
+ because `f2py -c` takes a list of SOURCES and compiles them itself. So a
1424
+ Fortran service could name a library and get nothing: `services/petro_api`
1425
+ bound the ten routines of its F90 facade, built an extension, and failed at
1426
+ import with `undefined symbol: iprvog_` — the seven F77 decks the facade
1427
+ calls into were never compiled.
1428
+
1429
+ The fix is the mechanism nativegate already uses everywhere else: the
1430
+ library's sources are expanded into `native/_expanded/` alongside the
1431
+ service's own and handed to f2py as ordinary sources. The original tree
1432
+ stays read-only, and the Docker build context stays the service directory,
1433
+ because everything the build needs has been copied under it.
1434
+
1435
+ INCLUDE directories are discovered rather than demanded. A legacy tree
1436
+ keeps its COMMON blocks in `include/*.INC` next to the decks; requiring the
1437
+ user to also spell that out in `include_paths:` would be asking them to
1438
+ restate something the library already says. Explicit `include_paths:` still
1439
+ win — these are added to them, not instead.
1440
+ """
1441
+ sources: list[Path] = []
1442
+ include_dirs: list[Path] = []
1443
+
1444
+ for name in config.libraries:
1445
+ lib_dir = Path(LIBRARIES_DIR) / name
1446
+ if not lib_dir.is_dir():
1447
+ raise click.ClickException(
1448
+ f"Service '{config.name}' declares library '{name}' but "
1449
+ f"{lib_dir} does not exist."
1450
+ )
1451
+
1452
+ # Deliberately NOT requiring a CMakeLists.txt the way the C++ path
1453
+ # does: that file declares a CMake target, and f2py never uses one.
1454
+ found = sorted(
1455
+ path
1456
+ for path in lib_dir.rglob("*")
1457
+ if path.suffix.lower() in (".f", ".f90", ".f77", ".for")
1458
+ and "_expanded" not in path.parts
1459
+ )
1460
+ if not found:
1461
+ raise click.ClickException(
1462
+ f"Service '{config.name}' is Fortran and declares library "
1463
+ f"'{name}', but {lib_dir} contains no Fortran sources. (A C++ "
1464
+ "library cannot be linked into an f2py extension.)"
1465
+ )
1466
+ sources += found
1467
+
1468
+ include_dirs += sorted(
1469
+ {
1470
+ path.parent
1471
+ for path in lib_dir.rglob("*")
1472
+ if path.suffix.lower() == ".inc"
1473
+ }
1474
+ )
1475
+
1476
+ return sources, include_dirs
1477
+
1478
+
1479
+ def _report_skipped(source: Path, module) -> None:
1480
+ """Print declarations the parser recognised but could not bind.
1481
+
1482
+ Silence here is the failure mode that costs the most time: a header with
1483
+ thirty methods generates a module with twenty-six and nothing says which
1484
+ four are missing or why, so the gap is discovered from an AttributeError
1485
+ in Python much later.
1486
+ """
1487
+ for message in getattr(module, "diagnostics", []):
1488
+ # The AST parser recovers from errors by inventing types (an unknown
1489
+ # return type becomes `int`), so a header that does not compile can
1490
+ # still produce plausible-looking bindings. Say so.
1491
+ click.echo(f" {source.name}: compiler error: {message}")
1492
+
1493
+ if not module.skipped:
1494
+ return
1495
+ click.echo(f" {source.name}: skipped {len(module.skipped)} declaration(s):")
1496
+ for entry in module.skipped:
1497
+ click.echo(f" - {entry.name}: {entry.reason}")
1498
+
1499
+
1500
+ def _has_undefined_methods(module: ModuleIR) -> bool:
1501
+ """True if anything bound needs a definition from a separate .cpp.
1502
+
1503
+ Structs are pure data and constructors of header-only classes may well be
1504
+ inline, so only methods and free functions count.
1505
+ """
1506
+ return any(cls.methods for cls in module.classes) or bool(module.functions)
1507
+
1508
+
1509
+ def _validated_include_paths(config: ServiceConfig) -> list[str]:
1510
+ """Extra header search directories, repo-root-relative.
1511
+
1512
+ For C++ these become `target_include_directories` entries so a source can
1513
+ `#include` a header living outside the service's own native/ directory.
1514
+ (Fortran uses the same config key for INCLUDE resolution.)
1515
+ """
1516
+ for p in config.include_paths:
1517
+ if not Path(p).is_dir():
1518
+ raise click.ClickException(
1519
+ f"include_paths entry '{p}' is not a directory (relative to the repo root)."
1520
+ )
1521
+ return list(config.include_paths)
1522
+
1523
+
1524
+ def _clang_options(clang: ClangConfig) -> ClangOptions:
1525
+ """nativegate.yaml's `clang:` block as parser options."""
1526
+ return ClangOptions(
1527
+ std=clang.std,
1528
+ include_paths=tuple(clang.include_paths),
1529
+ defines=tuple(clang.defines),
1530
+ extra_args=tuple(clang.extra_args),
1531
+ )
1532
+
1533
+
1534
+ def _generate_cpp_service(service_dir: Path, config: ServiceConfig) -> None:
1535
+ headers = find_native_sources(service_dir / "native", "cpp")
1536
+ headers = [h for h in headers if h.suffix in (".hpp", ".hh", ".h")]
1537
+ if not headers:
1538
+ raise click.ClickException(f"No C++ headers found under {service_dir / 'native'}.")
1539
+
1540
+ # All headers merge into ONE module. A Python extension can only carry a
1541
+ # single PYBIND11_MODULE init symbol, so a per-header bindings file would
1542
+ # leave every header but one orphaned — which is what happened before:
1543
+ # engine.hpp's bindings were generated, never compiled, and `Engine`
1544
+ # vanished from the package with no warning.
1545
+ merged = ModuleIR(
1546
+ name=config.name,
1547
+ language="cpp",
1548
+ source_file=str(service_dir / "native"),
1549
+ )
1550
+ contributing_headers: list[str] = []
1551
+ origin: dict[str, str] = {}
1552
+
1553
+ # A type forward-declared in one header may be defined in a sibling — all
1554
+ # of a service's headers compile into one extension, so it's complete by
1555
+ # the time pybind11 sees it. Collect the definitions first so those
1556
+ # symbols aren't wrongly skipped as incomplete.
1557
+ try:
1558
+ backend = cpp_parser.resolve_backend(config.parser)
1559
+ except cpp_parser.ParserUnavailable as exc:
1560
+ raise click.ClickException(str(exc)) from exc
1561
+ options = _clang_options(config.clang)
1562
+ click.echo(f"Parsing C++ with {cpp_parser.backend_description(config.parser)}.")
1563
+
1564
+ sibling_records = frozenset().union(
1565
+ *(
1566
+ cpp_parser.defined_record_names(h, backend=backend, options=options)
1567
+ for h in headers
1568
+ )
1569
+ )
1570
+
1571
+ for header in headers:
1572
+ try:
1573
+ module = cpp_parser.parse_header(
1574
+ header,
1575
+ config.expose,
1576
+ sibling_records,
1577
+ backend=backend,
1578
+ options=options,
1579
+ )
1580
+ except NativeTypeError as exc:
1581
+ raise click.ClickException(str(exc)) from exc
1582
+
1583
+ _report_skipped(header, module)
1584
+
1585
+ if module.is_empty():
1586
+ continue
1587
+
1588
+ for symbol in [*module.classes, *module.structs, *module.functions]:
1589
+ previous = origin.get(symbol.name)
1590
+ if previous is not None:
1591
+ raise click.ClickException(
1592
+ f"'{symbol.name}' is defined in both {previous} and {header.name}. "
1593
+ "nativegate binds all headers of a service into one extension, so "
1594
+ "symbol names must be unique across them — rename one, or expose "
1595
+ "only one of the headers via `expose:` in nativegate.yaml."
1596
+ )
1597
+ origin[symbol.name] = header.name
1598
+
1599
+ merged.classes.extend(module.classes)
1600
+ merged.structs.extend(module.structs)
1601
+ merged.functions.extend(module.functions)
1602
+ merged.skipped.extend(module.skipped)
1603
+ contributing_headers.append(header.name)
1604
+
1605
+ if merged.is_empty():
1606
+ raise click.ClickException(
1607
+ f"No exposable symbols found in {service_dir / 'native'}. "
1608
+ "Check `expose:` in nativegate.yaml, or that the headers declare "
1609
+ "classes/functions nativegate can parse."
1610
+ )
1611
+
1612
+ if len(contributing_headers) > 1:
1613
+ click.echo(f"Merged {len(contributing_headers)} headers: {', '.join(contributing_headers)}")
1614
+
1615
+ bindings_dir = service_dir / "bindings" / "generated"
1616
+ bindings_dir.mkdir(parents=True, exist_ok=True)
1617
+ # Everything here is generated, so stale files from a previous run (a
1618
+ # renamed service, or the old one-file-per-header scheme) are dead weight
1619
+ # that CMake no longer references — clear them rather than leave misleading
1620
+ # source lying around.
1621
+ for stale in bindings_dir.glob("*_bindings.cpp"):
1622
+ stale.unlink()
1623
+ bindings_path = bindings_dir / f"{config.name}_bindings.cpp"
1624
+ bindings_path.write_text(pybind_gen.generate_bindings(merged, contributing_headers))
1625
+
1626
+ native_dir = service_dir / "native"
1627
+ impl_files = sorted(
1628
+ p for p in native_dir.iterdir() if p.suffix in (".cpp", ".cc", ".cxx")
1629
+ )
1630
+ if not impl_files and _has_undefined_methods(merged):
1631
+ # Nothing links these declarations. On macOS the extension still
1632
+ # *builds* — undefined symbols in a module bundle are resolved lazily
1633
+ # at dlopen — so the failure surfaces much later as
1634
+ # "symbol not found in flat namespace '__ZN9Simulator10advance_toEd'",
1635
+ # which points at the linker rather than the missing file.
1636
+ click.echo(
1637
+ f"WARNING: no .cpp/.cc/.cxx implementation files in {native_dir}, but the "
1638
+ "headers declare methods without inline bodies. `ngate build` will "
1639
+ "succeed and then fail on first import with an undefined-symbol error. "
1640
+ "Add the implementation file(s) and re-run `ngate generate "
1641
+ f"{config.name}`."
1642
+ )
1643
+
1644
+ native_sources = [f"native/{s.name}" for s in impl_files]
1645
+ cmake_path = service_dir / "CMakeLists.txt"
1646
+ cmake_path.write_text(
1647
+ cmake_gen.generate_cmake(
1648
+ merged,
1649
+ config.name,
1650
+ native_sources + [f"bindings/generated/{bindings_path.name}"],
1651
+ libraries=_validated_libraries(config),
1652
+ include_paths=_validated_include_paths(config),
1653
+ )
1654
+ )
1655
+
1656
+ _write_package(service_dir, config, merged)
1657
+
1658
+
1659
+ def _with_shims(text: str, shims: list[str], source: Path) -> str:
1660
+ """Insert generated flattening shims into a module's `_expanded` copy.
1661
+
1662
+ Before `end module`, because the shim constructs the derived type and that
1663
+ type is only visible from inside the module that defines it. This edits
1664
+ the GENERATED copy only — the original tree stays read-only, same rule as
1665
+ every other transform under `_expanded/`.
1666
+ """
1667
+ if not shims:
1668
+ return text
1669
+ import re as _re
1670
+
1671
+ match = None
1672
+ for match in _re.finditer(r"(?im)^\s*end\s+module\b.*$", text):
1673
+ pass # keep the last one
1674
+ if match is None:
1675
+ raise click.ClickException(
1676
+ f"{source.name}: routines here need a flattening shim, but no "
1677
+ "`end module` line was found to anchor it. This is a nativegate "
1678
+ "bug — please report it with the source layout."
1679
+ )
1680
+ insertion = "\n" + "\n\n".join(shims) + "\n\n"
1681
+ return text[: match.start()] + insertion + text[match.start() :]
1682
+
1683
+
1684
+ def _generate_fortran_service(service_dir: Path, config: ServiceConfig) -> None:
1685
+ if not config.expose.functions:
1686
+ raise click.ClickException(
1687
+ "Fortran services require an `expose.functions:` list in nativegate.yaml "
1688
+ "naming exactly the routines to bind — there is no expose-everything "
1689
+ "fallback, so large legacy templates only pay for what they use. "
1690
+ "Add e.g.:\n expose:\n functions:\n - calculate_pressure"
1691
+ )
1692
+
1693
+ sources = find_native_sources(service_dir / "native", "fortran")
1694
+ if not sources:
1695
+ raise click.ClickException(f"No Fortran sources found under {service_dir / 'native'}.")
1696
+
1697
+ include_paths = [Path(p) for p in config.include_paths]
1698
+ for p in include_paths:
1699
+ if not p.is_dir():
1700
+ raise click.ClickException(
1701
+ f"include_paths entry '{p}' is not a directory (relative to the repo root)."
1702
+ )
1703
+
1704
+ # Preprocessed Fortran (.F90/.F, or #ifdef in a lowercase file) is not
1705
+ # Fortran yet — it is INPUT to the C preprocessor, and parsing it directly
1706
+ # reads every conditional branch as simultaneously live. It used to be
1707
+ # warned about and mis-read; it is now preprocessed with gfortran's own
1708
+ # `-cpp -E` into native/_expanded/ BEFORE anything parses it, so discovery,
1709
+ # intent inference and f2py all see the code gfortran would have compiled.
1710
+ preprocessed_set: set[Path] = set()
1711
+ needs_cpp = [s for s in sources if requires_preprocessing(s)]
1712
+ if needs_cpp:
1713
+ expanded_dir = service_dir / "native" / "_expanded"
1714
+ expanded_dir.mkdir(parents=True, exist_ok=True)
1715
+ replaced: list[Path] = []
1716
+ for source in sources:
1717
+ if source not in needs_cpp:
1718
+ replaced.append(source)
1719
+ continue
1720
+ target = expanded_dir / (source.stem + source.suffix.lower())
1721
+ try:
1722
+ target.write_text(
1723
+ run_c_preprocessor(
1724
+ source, include_paths, config.fortran_defines
1725
+ )
1726
+ )
1727
+ except PreprocessError as exc:
1728
+ raise click.ClickException(str(exc)) from exc
1729
+ replaced.append(target)
1730
+ preprocessed_set.add(target)
1731
+ sources = replaced
1732
+ click.echo(
1733
+ f"Preprocessed {len(needs_cpp)} source(s) with gfortran -cpp -E "
1734
+ f"-> {expanded_dir}"
1735
+ + (
1736
+ f" (defines: {', '.join(config.fortran_defines)})"
1737
+ if config.fortran_defines
1738
+ else ""
1739
+ )
1740
+ )
1741
+
1742
+ # Sources from `libraries:` are compiled INTO the extension — f2py links no
1743
+ # external target — so they join the service's own before anything below
1744
+ # classifies or expands them. See _fortran_library_inputs.
1745
+ library_sources, library_includes = _fortran_library_inputs(config)
1746
+ include_paths += [d for d in library_includes if d not in include_paths]
1747
+
1748
+ # A library file whose name matches one the service already has is the
1749
+ # facade-was-copied-in case: services/petro_api/native/petro_api.f90 is the
1750
+ # same routine as libraries/petro/fortran/modern/petro_api.f90. Compiling
1751
+ # both would be a duplicate-symbol link error, so the service's copy wins —
1752
+ # it is the one that was parsed and whose intents were inferred.
1753
+ own_names = {source.name for source in sources}
1754
+ shadowed = [lib for lib in library_sources if lib.name in own_names]
1755
+ library_sources = [lib for lib in library_sources if lib.name not in own_names]
1756
+ for lib in shadowed:
1757
+ click.echo(f"Using native/{lib.name} instead of {lib} (same file name).")
1758
+
1759
+ by_name: dict[str, Path] = {}
1760
+ for lib in library_sources:
1761
+ clash = by_name.get(lib.name)
1762
+ if clash is not None:
1763
+ raise click.ClickException(
1764
+ f"Two library sources are both named '{lib.name}' ({clash} and "
1765
+ f"{lib}). They would collide in native/_expanded/ and produce "
1766
+ "duplicate symbols. Rename one, or expose only one library."
1767
+ )
1768
+ by_name[lib.name] = lib
1769
+
1770
+ if library_sources:
1771
+ click.echo(
1772
+ f"Compiling {len(library_sources)} source(s) from "
1773
+ f"{', '.join(config.libraries)} into the extension."
1774
+ )
1775
+
1776
+ # A requested routine may live in any one of several native/*.f90 files
1777
+ # (common for large template libraries split across many files); search
1778
+ # each source until every requested name is found, rather than requiring
1779
+ # the caller to say which file it's in.
1780
+ remaining = list(config.expose.functions)
1781
+ module = None
1782
+ intents_by_source: dict[Path, dict[str, dict[str, tuple[str, bool]]]] = {}
1783
+ shims_by_source: dict[Path, list[str]] = {}
1784
+
1785
+ for source in sources:
1786
+ if not remaining:
1787
+ break
1788
+ found_expose = ExposeConfig(functions=[n for n in remaining if _routine_in_file(source, n)])
1789
+ if not found_expose.functions:
1790
+ continue
1791
+
1792
+ try:
1793
+ parsed = fortran_parser.parse_source(
1794
+ source, found_expose, include_paths=include_paths
1795
+ )
1796
+ except IncludeError as exc:
1797
+ raise click.ClickException(str(exc)) from exc
1798
+ shims_by_source[source] = list(parsed.fortran_shims)
1799
+ if module is None:
1800
+ module = parsed
1801
+ else:
1802
+ module.fortran_shims.extend(parsed.fortran_shims)
1803
+ # Routines from different Fortran modules (or from a module and
1804
+ # from bare fixed-form decks) can coexist: FunctionDef carries its
1805
+ # own enclosing module, and generate_init_py re-exports each from
1806
+ # wherever f2py actually put it.
1807
+ module.functions.extend(parsed.functions)
1808
+ module.skipped.extend(parsed.skipped)
1809
+
1810
+ # Remember which routines came from which file: the inferred intents
1811
+ # have to be written back into that file's expanded copy as Cf2py
1812
+ # directives, or f2py never learns about them.
1813
+ intents_by_source[source] = {
1814
+ fn.name: {p.name: (p.intent, p.is_array) for p in fn.parameters}
1815
+ for fn in parsed.functions
1816
+ }
1817
+
1818
+ for name in found_expose.functions:
1819
+ remaining.remove(name)
1820
+
1821
+ if remaining:
1822
+ raise click.ClickException(
1823
+ f"Could not find routine(s) {', '.join(remaining)} in any file under {service_dir / 'native'}."
1824
+ )
1825
+
1826
+ assert module is not None
1827
+ module.name = config.name
1828
+
1829
+ # f2py silently mis-wraps routines containing an in-body INCLUDE — the
1830
+ # extension builds and imports but arguments never arrive, so calls
1831
+ # return uninitialized memory. Hand it pre-expanded source instead.
1832
+ # See preprocess.expand_includes for the verification of this.
1833
+ # Free-form sources get the same treatment for a different reason: f2py's
1834
+ # generated wrapper does not inherit the module's kind PARAMETERs, so
1835
+ # `real(dp)` has to be resolved to `real(8)` before it reaches f2py.
1836
+ # See preprocess.resolve_kind_parameters.
1837
+ # A source needing the C preprocessor (an uppercase .F90/.F suffix, or cpp
1838
+ # directives in the text) is parsed here as if every #ifdef branch were
1839
+ # live, because neither Fortran parser has a preprocessor. That silently
1840
+ # binds whichever branch happens to lex, so say so rather than let it pass.
1841
+ # From here on `sources` means "everything that gets compiled", not "the
1842
+ # files searched for exposed routines" — the routine search above is
1843
+ # deliberately limited to the service's own native/ directory.
1844
+ sources = sources + library_sources
1845
+
1846
+ for source in sources:
1847
+ directives = find_cpp_directives(read_source(source))
1848
+ if requires_preprocessing(source):
1849
+ click.echo(
1850
+ f"WARNING: {source.name} needs the C preprocessor"
1851
+ + (f" (found {', '.join('#' + d for d in directives[:4])})" if directives else "")
1852
+ + ". nativegate has no preprocessor, so conditional code is read "
1853
+ "as though every branch is active. Pre-process it yourself and "
1854
+ "point nativegate at the output if the branches differ."
1855
+ )
1856
+
1857
+ fixed_form_sources = [s for s in sources if is_fixed_form(s)]
1858
+ # A6: an INCLUDE in a .f90 has exactly the same consequence as one in a
1859
+ # fixed-form deck — f2py mis-wraps the routine and calls return
1860
+ # uninitialized memory — so free-form sources get expanded too.
1861
+ free_form_include_sources = [
1862
+ s for s in sources if s not in fixed_form_sources and _has_include(s)
1863
+ ]
1864
+ kind_param_sources = [
1865
+ s
1866
+ for s in sources
1867
+ if s not in fixed_form_sources
1868
+ and s not in free_form_include_sources
1869
+ and uses_kind_parameters(read_source(s))
1870
+ ]
1871
+ # A source whose routines needed a flattening shim must be copied so the
1872
+ # shim has somewhere to live — even when nothing else forces a rewrite.
1873
+ shim_only_sources = [
1874
+ s
1875
+ for s in sources
1876
+ if shims_by_source.get(s)
1877
+ and s not in fixed_form_sources
1878
+ and s not in free_form_include_sources
1879
+ and s not in kind_param_sources
1880
+ ]
1881
+
1882
+ rewritten = (
1883
+ fixed_form_sources
1884
+ + free_form_include_sources
1885
+ + kind_param_sources
1886
+ + shim_only_sources
1887
+ )
1888
+ # A library source is always copied under the service, even when it needs
1889
+ # no rewriting: it lives outside native/, so referencing it as
1890
+ # `native/<name>` would name a file that is not there — and the Docker
1891
+ # build context is the service directory, so a path outside it could not be
1892
+ # COPYed in anyway.
1893
+ library_set = set(library_sources)
1894
+ if rewritten or library_sources or preprocessed_set:
1895
+ expanded_dir = service_dir / "native" / "_expanded"
1896
+ expanded_dir.mkdir(parents=True, exist_ok=True)
1897
+ native_sources = []
1898
+ for source in sources:
1899
+ if source in fixed_form_sources:
1900
+ target = expanded_dir / source.name
1901
+ expanded = expand_includes(source, include_paths)
1902
+ expanded = fixed_form.inject_intent_directives(
1903
+ expanded, intents_by_source.get(source, {})
1904
+ )
1905
+ target.write_text(expanded)
1906
+ native_sources.append(f"native/_expanded/{source.name}")
1907
+ elif source in free_form_include_sources:
1908
+ target = expanded_dir / source.name
1909
+ expanded = expand_includes(
1910
+ source, include_paths, comment_style="free"
1911
+ )
1912
+ # An expanded INCLUDE routinely carries the kind PARAMETERs the
1913
+ # body needs, so resolve them on the same copy.
1914
+ target.write_text(resolve_kind_parameters(expanded))
1915
+ native_sources.append(f"native/_expanded/{source.name}")
1916
+ elif source in kind_param_sources:
1917
+ target = expanded_dir / source.name
1918
+ target.write_text(
1919
+ _with_shims(
1920
+ resolve_kind_parameters(read_source(source)),
1921
+ shims_by_source.get(source, []),
1922
+ source,
1923
+ )
1924
+ )
1925
+ native_sources.append(f"native/_expanded/{source.name}")
1926
+ elif source in shim_only_sources:
1927
+ target = expanded_dir / source.name
1928
+ target.write_text(
1929
+ _with_shims(
1930
+ read_source(source), shims_by_source[source], source
1931
+ )
1932
+ )
1933
+ native_sources.append(f"native/_expanded/{source.name}")
1934
+ elif source in library_set:
1935
+ target = expanded_dir / source.name
1936
+ target.write_text(read_source(source))
1937
+ native_sources.append(f"native/_expanded/{source.name}")
1938
+ elif source in preprocessed_set:
1939
+ # Already written under _expanded/ by the preprocessor pass.
1940
+ native_sources.append(f"native/_expanded/{source.name}")
1941
+ else:
1942
+ native_sources.append(f"native/{source.name}")
1943
+ if fixed_form_sources:
1944
+ click.echo(
1945
+ f"Expanded INCLUDEs for {len(fixed_form_sources)} fixed-form source(s) "
1946
+ f"-> {expanded_dir}"
1947
+ )
1948
+ if free_form_include_sources:
1949
+ click.echo(
1950
+ f"Expanded INCLUDEs for {len(free_form_include_sources)} free-form "
1951
+ f"source(s) -> {expanded_dir}"
1952
+ )
1953
+ if kind_param_sources:
1954
+ click.echo(
1955
+ f"Resolved kind parameters (real(dp) -> real(8)) for "
1956
+ f"{len(kind_param_sources)} source(s) -> {expanded_dir}"
1957
+ )
1958
+ else:
1959
+ native_sources = [f"native/{s.name}" for s in sources]
1960
+
1961
+ cmake_path = service_dir / "CMakeLists.txt"
1962
+ cmake_path.write_text(
1963
+ f2py_gen.generate_cmake(
1964
+ module,
1965
+ config.name,
1966
+ native_sources,
1967
+ # The shim name where one exists: f2py must wrap the flattened
1968
+ # entry point, not the original whose derived argument it cannot
1969
+ # express.
1970
+ only_routines=[fn.cpp_name or fn.name for fn in module.functions],
1971
+ )
1972
+ )
1973
+
1974
+ _write_package(service_dir, config, module)
1975
+
1976
+
1977
+ import re as _re
1978
+
1979
+ _INCLUDE_LINE_RE = _re.compile(r"^\s*INCLUDE\s+['\"][^'\"]+['\"]", _re.IGNORECASE | _re.MULTILINE)
1980
+
1981
+
1982
+ def _has_include(source: Path) -> bool:
1983
+ return _INCLUDE_LINE_RE.search(read_source(source)) is not None
1984
+
1985
+
1986
+ def _routine_in_file(source: Path, name: str) -> bool:
1987
+ import re
1988
+
1989
+ # The prefix can be several words ("DOUBLE PRECISION FUNCTION PVTRS"),
1990
+ # and fixed-form continuation means the "(" may not be on this line, so
1991
+ # don't require it.
1992
+ pattern = re.compile(
1993
+ rf"^[ \t]*(?:[A-Za-z0-9_*]+[ \t]+)*?(?:function|subroutine)[ \t]+{re.escape(name)}\b",
1994
+ re.IGNORECASE | re.MULTILINE,
1995
+ )
1996
+ return pattern.search(read_source(source)) is not None
1997
+
1998
+
1999
+ def _run(cmd: list[str], cwd: Path, log=click.echo) -> None:
2000
+ log(f"$ {' '.join(cmd)} (in {cwd})")
2001
+ result = subprocess.run(cmd, cwd=cwd)
2002
+ if result.returncode != 0:
2003
+ sys.exit(result.returncode)
2004
+
2005
+
2006
+ if __name__ == "__main__":
2007
+ main()