nativegate 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nativegate/__init__.py +1 -0
- nativegate/__main__.py +4 -0
- nativegate/buildinfo.py +344 -0
- nativegate/cli.py +2007 -0
- nativegate/config.py +991 -0
- nativegate/declared_invariants.py +565 -0
- nativegate/discovery.py +167 -0
- nativegate/driverbuild.py +626 -0
- nativegate/drivers/__init__.py +5 -0
- nativegate/drivers/cpp.py +616 -0
- nativegate/drivers/fortran.py +507 -0
- nativegate/generators/__init__.py +0 -0
- nativegate/generators/cmake_gen.py +101 -0
- nativegate/generators/docker_gen.py +614 -0
- nativegate/generators/error_gen.py +104 -0
- nativegate/generators/f2py_gen.py +91 -0
- nativegate/generators/gateway_gen.py +110 -0
- nativegate/generators/golden_gen.py +50 -0
- nativegate/generators/k8s_gen.py +212 -0
- nativegate/generators/mcp_gen.py +281 -0
- nativegate/generators/middleware_gen.py +717 -0
- nativegate/generators/pybind_gen.py +406 -0
- nativegate/generators/pyproject_gen.py +61 -0
- nativegate/generators/python_pkg_gen.py +1164 -0
- nativegate/generators/test_gen.py +160 -0
- nativegate/golden.py +747 -0
- nativegate/invariants.py +532 -0
- nativegate/ir.py +789 -0
- nativegate/lattice.py +350 -0
- nativegate/locking.py +216 -0
- nativegate/oracle.py +904 -0
- nativegate/parsers/__init__.py +0 -0
- nativegate/parsers/cpp.py +105 -0
- nativegate/parsers/cpp_ast.py +1652 -0
- nativegate/parsers/cpp_regex.py +812 -0
- nativegate/parsers/fixed_form.py +868 -0
- nativegate/parsers/fortran.py +157 -0
- nativegate/parsers/fortran_fparser.py +1116 -0
- nativegate/parsers/fortran_regex.py +686 -0
- nativegate/preprocess.py +335 -0
- nativegate/structural_invariants.py +762 -0
- nativegate/suggest.py +208 -0
- nativegate/templates/__init__.py +20 -0
- nativegate/templates/golden_test_template.py +248 -0
- nativegate/wire.py +438 -0
- nativegate-0.1.0.dist-info/METADATA +547 -0
- nativegate-0.1.0.dist-info/RECORD +50 -0
- nativegate-0.1.0.dist-info/WHEEL +5 -0
- nativegate-0.1.0.dist-info/entry_points.txt +3 -0
- nativegate-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,614 @@
|
|
|
1
|
+
"""Multi-stage Dockerfile generator (design.md section 11).
|
|
2
|
+
|
|
3
|
+
Also emits the CycloneDX SBOM for a generated service — see `generate_sbom`.
|
|
4
|
+
Both live here because both are about what actually ends up in the shipped
|
|
5
|
+
image, and both have to be reproducible to be worth anything.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import uuid
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Iterable, Sequence
|
|
15
|
+
|
|
16
|
+
# --- Base image pin -------------------------------------------------------
|
|
17
|
+
#
|
|
18
|
+
# WHY A DIGEST AND NOT A TAG
|
|
19
|
+
# `python:3.12-slim` is a MUTABLE tag. Docker Hub re-points it at a new image
|
|
20
|
+
# on every patch/security rebuild, so two builds a month apart silently get
|
|
21
|
+
# different interpreters, different libc, different libgfortran — with no
|
|
22
|
+
# record of the change anywhere in the build. For a product whose entire
|
|
23
|
+
# claim is "the re-hosted numerics return the same answers", an unpinned base
|
|
24
|
+
# is a hole straight through the middle of that claim. A digest is immutable:
|
|
25
|
+
# the same digest is byte-identical forever, or the pull fails.
|
|
26
|
+
#
|
|
27
|
+
# The digest below is a multi-arch OCI image index, so it still resolves
|
|
28
|
+
# correctly on amd64 and arm64 — pinning does not cost you cross-platform
|
|
29
|
+
# builds.
|
|
30
|
+
#
|
|
31
|
+
# HOW TO REFRESH IT (deliberately — this is a reviewed change, not a chore)
|
|
32
|
+
# With Docker available:
|
|
33
|
+
# docker pull python:3.12-slim
|
|
34
|
+
# docker inspect --format='{{index .RepoDigests 0}}' python:3.12-slim
|
|
35
|
+
# Without Docker, straight from the registry HTTP API:
|
|
36
|
+
# TOKEN=$(curl -s "https://auth.docker.io/token\
|
|
37
|
+
#?service=registry.docker.io&scope=repository:library/python:pull" \
|
|
38
|
+
# | python3 -c "import sys,json;print(json.load(sys.stdin)['token'])")
|
|
39
|
+
# curl -sI -H "Authorization: Bearer $TOKEN" \
|
|
40
|
+
# -H "Accept: application/vnd.oci.image.index.v1+json" \
|
|
41
|
+
# https://registry-1.docker.io/v2/library/python/manifests/3.12-slim \
|
|
42
|
+
# | grep -i docker-content-digest
|
|
43
|
+
# Then update BASE_IMAGE_DIGEST, note the date, rebuild, and re-run the
|
|
44
|
+
# generated service's golden-value tests before shipping the new base.
|
|
45
|
+
#
|
|
46
|
+
# Resolved 2026-08-22 from registry-1.docker.io (OCI image index) for
|
|
47
|
+
# python:3.12-slim.
|
|
48
|
+
BASE_IMAGE = "python:3.12-slim"
|
|
49
|
+
BASE_IMAGE_DIGEST = "sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a"
|
|
50
|
+
BASE_IMAGE_REF = f"{BASE_IMAGE}@{BASE_IMAGE_DIGEST}"
|
|
51
|
+
|
|
52
|
+
# --- apt pins -------------------------------------------------------------
|
|
53
|
+
#
|
|
54
|
+
# WHY THE VERSIONS ARE SPELLED OUT
|
|
55
|
+
# `apt-get install gfortran` resolves against the LIVE Debian archive, so the
|
|
56
|
+
# same Dockerfile, from the same digest-pinned base, installs a different
|
|
57
|
+
# compiler the week after Debian publishes one. The image digest pins the
|
|
58
|
+
# filesystem you start from; it does nothing about what you then download
|
|
59
|
+
# into it. For a tool whose whole claim is "the re-hosted numerics return the
|
|
60
|
+
# same answers", the compiler is the single most consequential input, and it
|
|
61
|
+
# was the one thing nothing pinned.
|
|
62
|
+
#
|
|
63
|
+
# These are the candidates in the base image named by BASE_IMAGE_DIGEST
|
|
64
|
+
# (Debian trixie), read from the image itself rather than from a changelog.
|
|
65
|
+
#
|
|
66
|
+
# WHEN A PIN GOES STALE
|
|
67
|
+
# Debian drops superseded versions from the live mirror, so an old pin
|
|
68
|
+
# eventually fails with "Version '...' for '...' was not found". That is the
|
|
69
|
+
# intended behaviour: a loud failure you resolve deliberately, rather than a
|
|
70
|
+
# silent toolchain change nobody reviewed. Refresh with:
|
|
71
|
+
#
|
|
72
|
+
# nativegate toolchain --refresh
|
|
73
|
+
#
|
|
74
|
+
# which reads the candidates out of the pinned base image and prints the
|
|
75
|
+
# block to paste here. Update the base digest and these pins together, then
|
|
76
|
+
# re-run the generated service's golden-value tests before shipping.
|
|
77
|
+
#
|
|
78
|
+
# WHY THE CONCRETE COMPILER PACKAGES ARE NAMED, NOT JUST `gfortran`
|
|
79
|
+
# Pinning `gfortran=4:14.2.0-1` pins a METAPACKAGE. It exists to provide the
|
|
80
|
+
# /usr/bin/gfortran symlink and to depend on the real compiler; the compiler
|
|
81
|
+
# itself is `gfortran-14`, and apt only honours `=version` for packages named
|
|
82
|
+
# on the command line. Measured in the built image:
|
|
83
|
+
#
|
|
84
|
+
# gfortran 4:14.2.0-1 <- what the first version of this pinned
|
|
85
|
+
# gfortran-14 14.2.0-19 <- the compiler that actually runs
|
|
86
|
+
#
|
|
87
|
+
# Debian can ship gfortran-14 14.2.0-20 with the metapackage unchanged, and a
|
|
88
|
+
# build that looked pinned would quietly change compilers. So the concrete
|
|
89
|
+
# packages are named too. That hard-codes the GCC major version, which moves
|
|
90
|
+
# with the Debian release — refresh it together with the base image digest.
|
|
91
|
+
#
|
|
92
|
+
# Read 2026-08-23 from python:3.12-slim@sha256:2c941e86... (Debian trixie).
|
|
93
|
+
APT_SUITE = "trixie"
|
|
94
|
+
APT_PINS = {
|
|
95
|
+
"build-essential": "12.12",
|
|
96
|
+
"cmake": "3.31.6-2",
|
|
97
|
+
# Metapackage: provides the unsuffixed /usr/bin/gfortran.
|
|
98
|
+
"gfortran": "4:14.2.0-1",
|
|
99
|
+
# The compilers that actually run, and the runtimes they emit code against.
|
|
100
|
+
"gcc-14": "14.2.0-19",
|
|
101
|
+
"g++-14": "14.2.0-19",
|
|
102
|
+
"gfortran-14": "14.2.0-19",
|
|
103
|
+
"libgfortran5": "14.2.0-19",
|
|
104
|
+
"libstdc++6": "14.2.0-19",
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _pinned(package: str) -> str:
|
|
109
|
+
"""`name=version` when the version is known, bare name otherwise.
|
|
110
|
+
|
|
111
|
+
An unpinned name is not silently tolerated in the languages nativegate
|
|
112
|
+
generates for — every package in the tables below has an entry — but a
|
|
113
|
+
caller passing `extra_build_deps` may name something nativegate has never
|
|
114
|
+
seen, and refusing to emit a Dockerfile for it would be worse than
|
|
115
|
+
installing it unpinned and saying so in the SBOM.
|
|
116
|
+
"""
|
|
117
|
+
version = APT_PINS.get(package)
|
|
118
|
+
return f"{package}={version}" if version else package
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# Both the metapackage and the concrete compiler are listed: the first gives
|
|
122
|
+
# the unsuffixed command every build system looks for, the second is what apt
|
|
123
|
+
# can actually pin. See APT_PINS.
|
|
124
|
+
_LANGUAGE_BUILD_DEPS = {
|
|
125
|
+
"cpp": ["build-essential", "gcc-14", "g++-14", "cmake"],
|
|
126
|
+
"fortran": [
|
|
127
|
+
"build-essential", "gcc-14", "g++-14",
|
|
128
|
+
"gfortran", "gfortran-14", "cmake",
|
|
129
|
+
],
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
_LANGUAGE_RUNTIME_DEPS = {
|
|
133
|
+
"cpp": [],
|
|
134
|
+
"fortran": ["libgfortran5"],
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
# Extra pip-installed build tools, on top of `build`.
|
|
138
|
+
#
|
|
139
|
+
# WHY MESON IS NOT OPTIONAL FOR FORTRAN
|
|
140
|
+
# The generated CMake runs `python -m numpy.f2py -c`. distutils was removed
|
|
141
|
+
# in Python 3.12, so numpy's f2py falls back to its meson backend and shells
|
|
142
|
+
# out to a `meson` executable. BASE_IMAGE is python:3.12-slim. Without this,
|
|
143
|
+
# every generated Fortran image fails to build with
|
|
144
|
+
# `FileNotFoundError: [Errno 2] No such file or directory: 'meson'` — which
|
|
145
|
+
# is exactly how CI found it, on the first run that ever executed.
|
|
146
|
+
#
|
|
147
|
+
# Installed with pip rather than apt on purpose: Debian stable's `meson` is
|
|
148
|
+
# older than what f2py's backend expects, and pinning the toolchain matters
|
|
149
|
+
# more here than saving a layer. ninja comes along because meson drives it.
|
|
150
|
+
_LANGUAGE_PIP_BUILD_TOOLS = {
|
|
151
|
+
"fortran": ["meson", "ninja"],
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
# PEP 517 build requirements, installed explicitly so the wheel can be built
|
|
155
|
+
# with --no-build-isolation. Mirrors pyproject_gen._BUILD_REQUIRES; kept as a
|
|
156
|
+
# local copy for the same reason the runtime list is, so drift shows up as a
|
|
157
|
+
# diff in review.
|
|
158
|
+
#
|
|
159
|
+
# WHY --no-build-isolation, WHICH IS THE OPPOSITE OF THE USUAL ADVICE
|
|
160
|
+
# Two reasons, and the build failed on the first one.
|
|
161
|
+
#
|
|
162
|
+
# 1. It did not work. `pip wheel .` runs the backend in an isolated
|
|
163
|
+
# environment, and f2py's meson backend then died with
|
|
164
|
+
# `ModuleNotFoundError: No module named 'mesonbuild'` — the `meson`
|
|
165
|
+
# executable was on PATH from the layer above, but the module it needs
|
|
166
|
+
# was not importable from inside the isolated build.
|
|
167
|
+
#
|
|
168
|
+
# 2. Isolation is itself a reproducibility hole. pip *downloads* the build
|
|
169
|
+
# requirements into that environment at build time, unpinned — so the
|
|
170
|
+
# scikit-build-core and numpy that compile the extension could differ
|
|
171
|
+
# between two builds of the same pinned Dockerfile, which is exactly what
|
|
172
|
+
# this whole exercise is closing. Installing them explicitly makes them
|
|
173
|
+
# visible, and lockable.
|
|
174
|
+
_LANGUAGE_BUILD_REQUIRES = {
|
|
175
|
+
"cpp": ["scikit-build-core", "pybind11"],
|
|
176
|
+
"fortran": ["scikit-build-core", "numpy"],
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
# --- Crash containment (ROADMAP 2.2) --------------------------------------
|
|
180
|
+
#
|
|
181
|
+
# Legacy numerical code outside its validated envelope segfaults, calls
|
|
182
|
+
# Fortran STOP, or hangs. None of those raise a Python exception; the process
|
|
183
|
+
# simply dies or stops answering, and no `except` clause anywhere can see it.
|
|
184
|
+
# A bare `uvicorn` is ONE process, so any of the three took the whole service
|
|
185
|
+
# down and left nothing running to restart it.
|
|
186
|
+
#
|
|
187
|
+
# THE TOPOLOGY THAT SHIPS: a supervised worker pool.
|
|
188
|
+
# gunicorn's master process supervises N uvicorn workers. A worker that
|
|
189
|
+
# segfaults or STOPs is reaped and respawned; the master keeps the listening
|
|
190
|
+
# socket, so the service never stops accepting. `--timeout 60` is the part
|
|
191
|
+
# that covers HANGS: gunicorn kills a worker that has not completed its
|
|
192
|
+
# request in time, which no amount of exception handling can do.
|
|
193
|
+
# `--max-requests` recycles workers periodically, bounding the damage from
|
|
194
|
+
# native code that leaks rather than crashes.
|
|
195
|
+
#
|
|
196
|
+
# WHAT IT DOES NOT DO: per-request isolation. Requests sharing the crashed
|
|
197
|
+
# worker die with it. Containing a crash to only the request that caused it
|
|
198
|
+
# needs process-per-call, which puts marshalling cost on every request; that
|
|
199
|
+
# is a separate, opt-in tier and is not implemented. Anything taking
|
|
200
|
+
# untrusted input needs it — see docs/production-readiness.md.
|
|
201
|
+
#
|
|
202
|
+
# WHY THE FORTRAN DEFAULT IS ONE WORKER, AND IT IS NOT A THROUGHPUT OVERSIGHT
|
|
203
|
+
# COMMON blocks are per-PROCESS storage. With one worker, a client's
|
|
204
|
+
# `/pvt_set_fluid` and its follow-up `/solution_gor` at least reach the same
|
|
205
|
+
# COMMON. With several, they can land on different processes and the compute
|
|
206
|
+
# reads a COMMON that was never configured — turning an interleaving race
|
|
207
|
+
# into a straightforwardly wrong answer. Raising WEB_CONCURRENCY is correct
|
|
208
|
+
# ONLY for routines that carry no state across calls. gunicorn reads
|
|
209
|
+
# WEB_CONCURRENCY itself, so this is an env change, not an image rebuild.
|
|
210
|
+
_LANGUAGE_DEFAULT_WORKERS = {
|
|
211
|
+
# Per-request instances, no process-global state the parser can see.
|
|
212
|
+
"cpp": 2,
|
|
213
|
+
# See above: >1 changes the semantics of stateful routines.
|
|
214
|
+
"fortran": 1,
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def generate_dockerfile(
|
|
219
|
+
service_name: str,
|
|
220
|
+
language: str,
|
|
221
|
+
package_name: str,
|
|
222
|
+
extra_build_deps: list[str] | None = None,
|
|
223
|
+
libraries: list[str] | None = None,
|
|
224
|
+
locked: bool = False,
|
|
225
|
+
needs_numpy: bool = False,
|
|
226
|
+
) -> str:
|
|
227
|
+
"""The service's multi-stage Dockerfile.
|
|
228
|
+
|
|
229
|
+
`locked` is set when `requirements.lock` exists beside the service (see
|
|
230
|
+
`ngate lock`). Without it the runtime `pip install` resolves the
|
|
231
|
+
service's dependencies from PyPI at build time, so an image rebuilt a week
|
|
232
|
+
later ships different Python code with no record of the change. With it,
|
|
233
|
+
the dependencies are installed by hash and the wheel goes in `--no-deps`.
|
|
234
|
+
"""
|
|
235
|
+
build_deps = _LANGUAGE_BUILD_DEPS.get(language, ["build-essential", "cmake"]) + (extra_build_deps or [])
|
|
236
|
+
runtime_deps = _LANGUAGE_RUNTIME_DEPS.get(language, [])
|
|
237
|
+
libraries = libraries or []
|
|
238
|
+
|
|
239
|
+
runtime_install = ""
|
|
240
|
+
if runtime_deps:
|
|
241
|
+
deps = " \\\n ".join(_pinned(d) for d in runtime_deps)
|
|
242
|
+
runtime_install = (
|
|
243
|
+
"\nRUN apt-get update && \\\n"
|
|
244
|
+
f" apt-get install -y --no-install-recommends {deps} && \\\n"
|
|
245
|
+
" rm -rf /var/lib/apt/lists/*\n"
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
build_deps_str = " \\\n ".join(_pinned(d) for d in build_deps)
|
|
249
|
+
pip_build_tools = " ".join(
|
|
250
|
+
["build"]
|
|
251
|
+
+ _LANGUAGE_BUILD_REQUIRES.get(language, _LANGUAGE_BUILD_REQUIRES["cpp"])
|
|
252
|
+
+ _LANGUAGE_PIP_BUILD_TOOLS.get(language, [])
|
|
253
|
+
+ (["numpy"] if language == "cpp" and needs_numpy else [])
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
# The lock lives beside the service, so the path it is COPYed from depends
|
|
257
|
+
# on which directory the build context is rooted at.
|
|
258
|
+
lock_source = (
|
|
259
|
+
f"services/{service_name}/requirements.lock" if libraries else "requirements.lock"
|
|
260
|
+
)
|
|
261
|
+
lock_block = (
|
|
262
|
+
f"""
|
|
263
|
+
COPY {lock_source} ./requirements.lock
|
|
264
|
+
|
|
265
|
+
# --require-hashes: every dependency pinned by version AND sha256, so a
|
|
266
|
+
# rebuild either installs exactly this or fails. Generated by `ngate lock`.
|
|
267
|
+
RUN pip install --no-cache-dir --require-hashes -r requirements.lock
|
|
268
|
+
"""
|
|
269
|
+
if locked
|
|
270
|
+
else ""
|
|
271
|
+
)
|
|
272
|
+
# Without the lock the wheel's dependencies must still be resolved, so
|
|
273
|
+
# --no-deps is only correct once they are already installed from it.
|
|
274
|
+
no_deps = " --no-deps" if locked else ""
|
|
275
|
+
|
|
276
|
+
workers = _LANGUAGE_DEFAULT_WORKERS.get(language, 1)
|
|
277
|
+
worker_note = (
|
|
278
|
+
"# One worker: COMMON blocks are per-process, so a configure call and\n"
|
|
279
|
+
"# the compute that reads it must reach the SAME process. Raise this\n"
|
|
280
|
+
"# only for routines that keep no state between calls.\n"
|
|
281
|
+
if language == "fortran"
|
|
282
|
+
else "# Per-request instances and no process-global state, so workers scale freely.\n"
|
|
283
|
+
)
|
|
284
|
+
crash_containment = (
|
|
285
|
+
"# Crash containment: gunicorn supervises the workers, so a segfault or\n"
|
|
286
|
+
"# Fortran STOP in native code kills one worker instead of the service,\n"
|
|
287
|
+
"# and --timeout reaps one that hangs. See docker_gen for the full note.\n"
|
|
288
|
+
f"{worker_note}"
|
|
289
|
+
f"ENV WEB_CONCURRENCY={workers}\n"
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
# A service that links shared libraries needs them inside the build
|
|
293
|
+
# context, and they live outside its own directory — so the context has
|
|
294
|
+
# to be the repo root and the paths become repo-relative. Without
|
|
295
|
+
# libraries we keep the simpler service-dir context.
|
|
296
|
+
if libraries:
|
|
297
|
+
context_note = (
|
|
298
|
+
f"# NOTE: build from the REPO ROOT, not services/{service_name}/:\n"
|
|
299
|
+
f"# docker build -f services/{service_name}/Dockerfile -t {service_name}:latest .\n"
|
|
300
|
+
f"# The shared libraries under libraries/ must be in the build context.\n"
|
|
301
|
+
)
|
|
302
|
+
copy_lines = "\n".join(
|
|
303
|
+
f"COPY libraries/{lib} ./libraries/{lib}" for lib in libraries
|
|
304
|
+
)
|
|
305
|
+
build_stage = f"""WORKDIR /build
|
|
306
|
+
|
|
307
|
+
{copy_lines}
|
|
308
|
+
COPY services/{service_name} ./services/{service_name}
|
|
309
|
+
|
|
310
|
+
WORKDIR /build/services/{service_name}
|
|
311
|
+
|
|
312
|
+
# A wheel is a zip, and a zip records a modification time per entry, so two
|
|
313
|
+
# builds of identical sources produced wheels with different SHA-256s. The
|
|
314
|
+
# compiled extension inside was byte-identical both times — measured — but the
|
|
315
|
+
# artifact you actually ship and sign was not. SOURCE_DATE_EPOCH is the
|
|
316
|
+
# reproducible-builds convention every Python packaging tool honours for
|
|
317
|
+
# exactly this. The value is arbitrary and fixed; 1980-01-01 is the earliest a
|
|
318
|
+
# zip can represent.
|
|
319
|
+
ENV SOURCE_DATE_EPOCH=315532800
|
|
320
|
+
|
|
321
|
+
RUN pip install --no-cache-dir {pip_build_tools}
|
|
322
|
+
|
|
323
|
+
# --no-deps, and it is load-bearing: without it `pip wheel` downloads every
|
|
324
|
+
# DEPENDENCY into /dist as well, and the `pip install /dist/*.whl` below then
|
|
325
|
+
# installs those instead of the hash-locked ones — silently defeating the lock.
|
|
326
|
+
# Caught by building the image and reading which numpy actually landed.
|
|
327
|
+
RUN pip wheel . -w /dist --no-build-isolation --no-deps
|
|
328
|
+
"""
|
|
329
|
+
else:
|
|
330
|
+
context_note = (
|
|
331
|
+
f"# Build from services/{service_name}/:\n"
|
|
332
|
+
f"# docker build -t {service_name}:latest .\n"
|
|
333
|
+
)
|
|
334
|
+
build_stage = f"""WORKDIR /build
|
|
335
|
+
|
|
336
|
+
COPY . .
|
|
337
|
+
|
|
338
|
+
# A wheel is a zip, and a zip records a modification time per entry, so two
|
|
339
|
+
# builds of identical sources produced wheels with different SHA-256s. The
|
|
340
|
+
# compiled extension inside was byte-identical both times — measured — but the
|
|
341
|
+
# artifact you actually ship and sign was not. SOURCE_DATE_EPOCH is the
|
|
342
|
+
# reproducible-builds convention every Python packaging tool honours for
|
|
343
|
+
# exactly this. The value is arbitrary and fixed; 1980-01-01 is the earliest a
|
|
344
|
+
# zip can represent.
|
|
345
|
+
ENV SOURCE_DATE_EPOCH=315532800
|
|
346
|
+
|
|
347
|
+
RUN pip install --no-cache-dir {pip_build_tools}
|
|
348
|
+
|
|
349
|
+
# --no-deps, and it is load-bearing: without it `pip wheel` downloads every
|
|
350
|
+
# DEPENDENCY into /dist as well, and the `pip install /dist/*.whl` below then
|
|
351
|
+
# installs those instead of the hash-locked ones — silently defeating the lock.
|
|
352
|
+
RUN pip wheel . -w /dist --no-build-isolation --no-deps
|
|
353
|
+
"""
|
|
354
|
+
|
|
355
|
+
return f"""# Generated by nativegate. Do not edit by hand — re-run `ngate docker {service_name}`.
|
|
356
|
+
{context_note}# Base image is DIGEST-PINNED, not tag-pinned — see docker_gen.BASE_IMAGE_DIGEST
|
|
357
|
+
# for why, and for how to refresh it.
|
|
358
|
+
FROM {BASE_IMAGE_REF} AS builder
|
|
359
|
+
|
|
360
|
+
RUN apt-get update && \\
|
|
361
|
+
apt-get install -y \\
|
|
362
|
+
{build_deps_str} && \\
|
|
363
|
+
rm -rf /var/lib/apt/lists/*
|
|
364
|
+
|
|
365
|
+
{build_stage}
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
FROM {BASE_IMAGE_REF}
|
|
369
|
+
{runtime_install}
|
|
370
|
+
WORKDIR /app
|
|
371
|
+
|
|
372
|
+
COPY --from=builder /dist /dist
|
|
373
|
+
{lock_block}
|
|
374
|
+
RUN pip install --no-cache-dir{no_deps} /dist/*.whl
|
|
375
|
+
|
|
376
|
+
# design.md section 21: run as non-root. UID is explicit so a Kubernetes
|
|
377
|
+
# `runAsUser: 1000` and `runAsNonRoot: true` match the image rather than
|
|
378
|
+
# fighting it.
|
|
379
|
+
RUN useradd --create-home --uid 1000 appuser
|
|
380
|
+
USER appuser
|
|
381
|
+
|
|
382
|
+
# The generated Kubernetes manifests set readOnlyRootFilesystem: true. Python
|
|
383
|
+
# writing .pyc files next to the source would fail against that, so the
|
|
384
|
+
# interpreter is told not to. PYTHONUNBUFFERED so logs reach the collector as
|
|
385
|
+
# they happen rather than when a buffer fills — a service that crashes with
|
|
386
|
+
# its last log lines still buffered is one you cannot debug.
|
|
387
|
+
ENV PYTHONDONTWRITEBYTECODE=1 \\
|
|
388
|
+
PYTHONUNBUFFERED=1
|
|
389
|
+
|
|
390
|
+
HEALTHCHECK --interval=30s --timeout=3s CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/docs')" || exit 1
|
|
391
|
+
|
|
392
|
+
{crash_containment}
|
|
393
|
+
CMD ["gunicorn", "{package_name}.service:app", \\
|
|
394
|
+
"--worker-class", "uvicorn_worker.UvicornWorker", \\
|
|
395
|
+
"--bind", "0.0.0.0:8000", \\
|
|
396
|
+
"--timeout", "60", \\
|
|
397
|
+
"--graceful-timeout", "30", \\
|
|
398
|
+
"--max-requests", "1000", \\
|
|
399
|
+
"--max-requests-jitter", "100"]
|
|
400
|
+
"""
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
# --- CycloneDX SBOM -------------------------------------------------------
|
|
404
|
+
|
|
405
|
+
SBOM_FILENAME = "sbom.cdx.json"
|
|
406
|
+
|
|
407
|
+
# CycloneDX 1.5 is the last spec revision with broad tool support (Grype,
|
|
408
|
+
# Trivy, Dependency-Track, the GitHub dependency graph). Bump deliberately.
|
|
409
|
+
CYCLONEDX_SPEC_VERSION = "1.5"
|
|
410
|
+
|
|
411
|
+
# Namespace for deriving the BOM serial number from the BOM's own content.
|
|
412
|
+
# CycloneDX wants a `serialNumber`, but a random UUID would make two SBOMs of
|
|
413
|
+
# identical inputs differ, which destroys the point of diffing them in review.
|
|
414
|
+
# A UUIDv5 over the content is stable AND still unique per distinct BOM.
|
|
415
|
+
_SERIAL_NAMESPACE = uuid.UUID("6ba7b811-9dad-11d1-80b4-00c04fd430c8") # RFC 4122 URL ns
|
|
416
|
+
|
|
417
|
+
# Runtime Python dependencies of a generated service, mirroring
|
|
418
|
+
# pyproject_gen's [project.dependencies]. Kept as a local copy on purpose:
|
|
419
|
+
# the SBOM should record what the generated pyproject actually declares, and
|
|
420
|
+
# a silent drift between the two is something a reviewer should see as a diff.
|
|
421
|
+
_LANGUAGE_RUNTIME_PYTHON_DEPS = {
|
|
422
|
+
"cpp": ["fastapi", "uvicorn", "gunicorn", "uvicorn-worker"],
|
|
423
|
+
"fortran": ["fastapi", "uvicorn", "gunicorn", "uvicorn-worker", "numpy"],
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
# Files the SBOM treats as "native sources compiled in". A wheel wrapping a
|
|
427
|
+
# 1988 Fortran deck is only auditable if the deck itself is in the manifest.
|
|
428
|
+
_NATIVE_SOURCE_SUFFIXES = {
|
|
429
|
+
".f", ".for", ".f77", ".f90", ".f95", ".f03", ".f08", ".inc",
|
|
430
|
+
".c", ".h", ".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx",
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _sha256_file(path: Path) -> str:
|
|
435
|
+
digest = hashlib.sha256()
|
|
436
|
+
with path.open("rb") as handle:
|
|
437
|
+
for chunk in iter(lambda: handle.read(1 << 16), b""):
|
|
438
|
+
digest.update(chunk)
|
|
439
|
+
return digest.hexdigest()
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def find_native_sources(root: Path) -> list[Path]:
|
|
443
|
+
"""Every native source under `root`, sorted, build output excluded.
|
|
444
|
+
|
|
445
|
+
Sorted so the SBOM component order does not depend on filesystem
|
|
446
|
+
iteration order — a directory listing is not stable across machines.
|
|
447
|
+
"""
|
|
448
|
+
root = Path(root)
|
|
449
|
+
skip = {"build", "dist", "_skbuild", ".git", "__pycache__", ".venv"}
|
|
450
|
+
found = [
|
|
451
|
+
path
|
|
452
|
+
for path in root.rglob("*")
|
|
453
|
+
if path.is_file()
|
|
454
|
+
and path.suffix.lower() in _NATIVE_SOURCE_SUFFIXES
|
|
455
|
+
and not any(part in skip for part in path.relative_to(root).parts)
|
|
456
|
+
]
|
|
457
|
+
return sorted(found, key=lambda p: p.relative_to(root).as_posix())
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def generate_sbom(
|
|
461
|
+
service_name: str,
|
|
462
|
+
language: str,
|
|
463
|
+
package_name: str,
|
|
464
|
+
*,
|
|
465
|
+
version: str = "1.0.0",
|
|
466
|
+
native_sources: Iterable[Path] | None = None,
|
|
467
|
+
source_root: Path | None = None,
|
|
468
|
+
libraries: Sequence[str] | None = None,
|
|
469
|
+
python_dependencies: Sequence[str] | None = None,
|
|
470
|
+
) -> str:
|
|
471
|
+
"""A deterministic CycloneDX 1.5 SBOM (JSON) for one generated service.
|
|
472
|
+
|
|
473
|
+
Records BOTH halves of what ships:
|
|
474
|
+
|
|
475
|
+
* the Python dependency surface (fastapi, uvicorn, numpy), and
|
|
476
|
+
* every native source compiled into the extension, each with its
|
|
477
|
+
SHA-256 and its path relative to the service directory.
|
|
478
|
+
|
|
479
|
+
The second half is the reason this exists. A wheel is an opaque binary;
|
|
480
|
+
without the source digests there is no way to answer "which revision of
|
|
481
|
+
the Fortran deck is inside this image?" during an audit.
|
|
482
|
+
|
|
483
|
+
Determinism: no timestamps, no random serial number, sorted components.
|
|
484
|
+
Two runs over unchanged inputs produce byte-identical output, so the SBOM
|
|
485
|
+
can be committed and diffed like any other reviewed artifact.
|
|
486
|
+
"""
|
|
487
|
+
source_root = Path(source_root) if source_root is not None else None
|
|
488
|
+
if native_sources is None:
|
|
489
|
+
native_sources = find_native_sources(source_root) if source_root else []
|
|
490
|
+
if python_dependencies is None:
|
|
491
|
+
python_dependencies = _LANGUAGE_RUNTIME_PYTHON_DEPS.get(
|
|
492
|
+
language, _LANGUAGE_RUNTIME_PYTHON_DEPS["cpp"]
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
components: list[dict] = []
|
|
496
|
+
|
|
497
|
+
# The build toolchain, which nothing recorded until now. The base image
|
|
498
|
+
# digest pinned the filesystem you start from and said nothing about what
|
|
499
|
+
# gets installed into it, so two images built a month apart could carry
|
|
500
|
+
# different compilers with byte-identical SBOMs — precisely the gap the
|
|
501
|
+
# golden values exist to close, left open in the artifact that is supposed
|
|
502
|
+
# to document it.
|
|
503
|
+
#
|
|
504
|
+
# Deterministic because APT_PINS is static. Scope distinguishes what ships
|
|
505
|
+
# from what only built it: the compilers stay in the builder stage
|
|
506
|
+
# ("excluded"), the runtime libraries are in the final image ("required").
|
|
507
|
+
build_only = set(_LANGUAGE_BUILD_DEPS.get(language, [])) - set(
|
|
508
|
+
_LANGUAGE_RUNTIME_DEPS.get(language, [])
|
|
509
|
+
)
|
|
510
|
+
apt_packages = sorted(
|
|
511
|
+
build_only | set(_LANGUAGE_RUNTIME_DEPS.get(language, []))
|
|
512
|
+
)
|
|
513
|
+
for name in apt_packages:
|
|
514
|
+
version = APT_PINS.get(name)
|
|
515
|
+
components.append(
|
|
516
|
+
{
|
|
517
|
+
"type": "library",
|
|
518
|
+
"bom-ref": f"deb/{name}",
|
|
519
|
+
"name": name,
|
|
520
|
+
**({"version": version} if version else {}),
|
|
521
|
+
"purl": (
|
|
522
|
+
f"pkg:deb/debian/{name}@{version}?distro={APT_SUITE}"
|
|
523
|
+
if version
|
|
524
|
+
else f"pkg:deb/debian/{name}?distro={APT_SUITE}"
|
|
525
|
+
),
|
|
526
|
+
"scope": "excluded" if name in build_only else "required",
|
|
527
|
+
}
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
for name in sorted(python_dependencies):
|
|
531
|
+
components.append(
|
|
532
|
+
{
|
|
533
|
+
"type": "library",
|
|
534
|
+
"bom-ref": f"pypi/{name}",
|
|
535
|
+
"name": name,
|
|
536
|
+
# No version: the generated pyproject.toml declares these
|
|
537
|
+
# unpinned, so claiming a version here would be a lie. Resolve
|
|
538
|
+
# the built image with `pip freeze`/syft to get versions, and
|
|
539
|
+
# see docs/deployment-topologies.md.
|
|
540
|
+
"purl": f"pkg:pypi/{name}",
|
|
541
|
+
"scope": "required",
|
|
542
|
+
}
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
for path in native_sources:
|
|
546
|
+
path = Path(path)
|
|
547
|
+
relative = (
|
|
548
|
+
path.relative_to(source_root).as_posix()
|
|
549
|
+
if source_root is not None and path.is_absolute()
|
|
550
|
+
else Path(path).as_posix()
|
|
551
|
+
)
|
|
552
|
+
components.append(
|
|
553
|
+
{
|
|
554
|
+
"type": "file",
|
|
555
|
+
"bom-ref": f"native-source/{relative}",
|
|
556
|
+
"name": relative,
|
|
557
|
+
"hashes": [{"alg": "SHA-256", "content": _sha256_file(path)}],
|
|
558
|
+
}
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
for library in sorted(libraries or []):
|
|
562
|
+
components.append(
|
|
563
|
+
{
|
|
564
|
+
"type": "library",
|
|
565
|
+
"bom-ref": f"native-library/{library}",
|
|
566
|
+
"name": library,
|
|
567
|
+
"description": (
|
|
568
|
+
"Shared native library from libraries/ linked statically "
|
|
569
|
+
"into this service's extension module."
|
|
570
|
+
),
|
|
571
|
+
}
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
document = {
|
|
575
|
+
"bomFormat": "CycloneDX",
|
|
576
|
+
"specVersion": CYCLONEDX_SPEC_VERSION,
|
|
577
|
+
"version": 1,
|
|
578
|
+
"metadata": {
|
|
579
|
+
# No "timestamp" key: it would change on every run and make the
|
|
580
|
+
# SBOM undiffable. CycloneDX makes it optional for exactly this.
|
|
581
|
+
"tools": [{"vendor": "nativegate", "name": "nativegate"}],
|
|
582
|
+
"component": {
|
|
583
|
+
"type": "application",
|
|
584
|
+
"bom-ref": f"pypi/{service_name}@{version}",
|
|
585
|
+
"name": service_name,
|
|
586
|
+
"version": version,
|
|
587
|
+
"purl": f"pkg:pypi/{service_name}@{version}",
|
|
588
|
+
"properties": [
|
|
589
|
+
{"name": "nativegate:language", "value": language},
|
|
590
|
+
{"name": "nativegate:package", "value": package_name},
|
|
591
|
+
{"name": "nativegate:baseImage", "value": BASE_IMAGE_REF},
|
|
592
|
+
],
|
|
593
|
+
},
|
|
594
|
+
},
|
|
595
|
+
"components": components,
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
# Serial number derived from the content above, so it is stable for
|
|
599
|
+
# identical inputs and changes when anything in the BOM changes.
|
|
600
|
+
payload = json.dumps(document, indent=2, sort_keys=True, ensure_ascii=False)
|
|
601
|
+
document["serialNumber"] = "urn:uuid:" + str(
|
|
602
|
+
uuid.uuid5(_SERIAL_NAMESPACE, payload)
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
return json.dumps(document, indent=2, ensure_ascii=False) + "\n"
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def write_sbom(service_dir: Path, service_name: str, language: str, package_name: str, **kwargs) -> Path:
|
|
609
|
+
"""Write `sbom.cdx.json` into the service directory and return its path."""
|
|
610
|
+
service_dir = Path(service_dir)
|
|
611
|
+
kwargs.setdefault("source_root", service_dir)
|
|
612
|
+
path = service_dir / SBOM_FILENAME
|
|
613
|
+
path.write_text(generate_sbom(service_name, language, package_name, **kwargs))
|
|
614
|
+
return path
|