nativegate 0.1.0__py3-none-any.whl

nativegate/generators/error_gen.py

@@ -0,0 +1,104 @@
+"""The generated apps' unhandled-exception handler.
+
+Lives in its own module because BOTH app generators need the identical block:
+`python_pkg_gen.generate_service_py` for the standalone container, and
+`gateway_gen.generate_gateway_app` for the composed topology. An exception
+handler is registered per FastAPI *app*, and a service router mounted into a
+gateway runs under the gateway's app — so a handler installed only on
+`service.py` would silently not apply in the composed deployment, which is the
+one more likely to be internet-facing. Emitting from one place is what stops
+those two paths drifting.
+
+WHAT THIS DOES AND DOES NOT COVER — measured, not assumed
+
+`docs/production-readiness.md` used to claim an uncaught exception returned "a
+raw traceback in the response". **That was wrong, and it was checked before
+this module was written.** With `debug=False` (FastAPI's default, and what the
+generated app uses), Starlette's ServerErrorMiddleware returns exactly
+`Internal Server Error` as `text/plain`. No traceback, no exception message,
+no paths. There was no information leak to close.
+
+So this handler is NOT a leak fix. What it actually buys:
+
+* **a correlatable id.** The bare 500 is untraceable: a caller reporting "it
+  returned 500 at about 2pm" cannot be matched to a log line. Every failure
+  now carries an `error_id` that appears in both the response and the log.
+* **a consistent body.** `text/plain` for failures and JSON for everything
+  else means a client needs two parsers. Errors are now JSON like the rest.
+* **a named logger.** `nativegate.<service>` can be raised or silenced without
+  touching the rest of the process.
+
+Two things it does not cover, both verified:
+
+* **`debug=True` still leaks the full traceback, and this handler does not
+  stop it** — Starlette checks `debug` *before* consulting the handler, so the
+  debug response wins. The protection against that is not enabling debug in a
+  deployed service; there is no code here that can override it.
+* **Python-level failures only** — an f2py argument error, a ValueError out of
+  a wrapper, a numpy conversion failure. Native code that segfaults, calls
+  Fortran `STOP`, or hangs takes the worker process down with it and never
+  reaches any Python handler. That is ROADMAP 2.2 (crash containment) and
+  needs process isolation, not a try/except.
+"""
+
+from __future__ import annotations
+
+# Imports the emitted block needs, on top of whatever the app module already
+# imports. Both call sites splice these in rather than hard-coding them, so
+# adding a dependency here cannot leave one generated app un-importable.
+# `Request` is deliberately NOT here: both app modules already import from
+# `fastapi` for `FastAPI`, and a second `from fastapi import ...` line would
+# be a lint finding in generated code nobody is allowed to edit by hand. The
+# call sites add `Request` to the import they already write.
+ERROR_HANDLER_IMPORTS = (
+    "import logging",
+    "import os",
+    "import uuid",
+    "",
+    "from fastapi.responses import JSONResponse",
+)
+
+
+def generate_error_handler(logger_name: str) -> str:
+    """The handler block, to be appended after `app = FastAPI(...)`.
+
+    `logger_name` names the logger so an operator can raise or silence one
+    service's errors without touching the rest of the process.
+    """
+    return f'''
+
+_LOG = logging.getLogger("nativegate.{logger_name}")
+
+# Off by default. A native routine's exception text can carry deck values,
+# array contents or absolute paths from the build machine; none of that should
+# leave the process just because something failed. Turn it on deliberately,
+# in a non-production environment, when a caller needs to see why.
+_DEBUG_ERRORS = os.environ.get("NATIVEGATE_DEBUG_ERRORS", "").lower() not in ("", "0", "false", "no")
+
+
+@app.exception_handler(Exception)
+async def _unhandled_native_error(request: Request, exc: Exception):
+    """Give a failed request a JSON body and an id that matches the log line.
+
+    Without this, Starlette answers `Internal Server Error` as plain text —
+    safe, but untraceable and a second content type for clients to parse.
+
+    HTTPException does NOT reach here — FastAPI handles it first — so the
+    generated argument-validation errors keep their own status and detail.
+
+    Caveats, both real: with `debug=True` Starlette returns its traceback
+    response *instead* of this one, so do not enable debug on a deployed
+    service; and a segfault, Fortran STOP or hang kills the worker process
+    before any Python handler runs. See docs/production-readiness.md.
+    """
+    error_id = uuid.uuid4().hex[:12]
+    # .exception() so the traceback reaches the log exactly once, keyed by the
+    # same id the caller was given — that pairing is the whole point.
+    _LOG.exception(
+        "unhandled error %s on %s %s", error_id, request.method, request.url.path
+    )
+    body = {{"error": "internal server error", "error_id": error_id}}
+    if _DEBUG_ERRORS:
+        body["detail"] = f"{{type(exc).__name__}}: {{exc}}"
+    return JSONResponse(status_code=500, content=body)
+'''

nativegate/generators/f2py_gen.py

@@ -0,0 +1,91 @@
+"""f2py-based CMake build generator (design.md sections 8, 14)."""
+
+from __future__ import annotations
+
+from ..ir import ModuleIR
+
+
+def generate_cmake(
+    module: ModuleIR,
+    service_name: str,
+    native_sources: list[str],
+    only_routines: list[str] | None = None,
+) -> str:
+    # add_custom_command's COMMAND runs with CMAKE_CURRENT_BINARY_DIR as its
+    # working directory, not the source dir — relative "native/foo.f90" only
+    # resolves by coincidence for in-source builds. scikit-build-core builds
+    # out-of-tree (a temp dir), so this must be an absolute path.
+    sources = "\n".join(f"    ${{CMAKE_CURRENT_SOURCE_DIR}}/{src}" for src in native_sources)
+
+    # `only: a b :` restricts f2py to the named routines. For legacy F77 this
+    # is a correctness requirement, not a size optimization: wrapping a whole
+    # file pulls in routines whose PARAMETER-dimensioned arrays f2py emits but
+    # never defines, and the C wrapper then fails to compile
+    # ("use of undeclared identifier 'mxcell'"). Verified against
+    # libraries/petro — matsol.f wraps fine per-routine, fails whole-file.
+    only_clause = ""
+    if only_routines:
+        only_clause = " only: " + " ".join(r.lower() for r in only_routines) + " :"
+
+    # f2py's intermediate build (custom module.c + optional f2pywrappers2.f90,
+    # reassembled by hand into python_add_library) only produces the wrapper
+    # file when routines actually need one (e.g. array-intent subroutines) —
+    # a plain `function` like a scalar-only routine doesn't generate it, so
+    # hardcoding it as a build output breaks for exactly that case. `f2py -c`
+    # runs f2py's own build end-to-end and sidesteps this entirely.
+    #
+    # WHY `--backend meson` IS PASSED EXPLICITLY RATHER THAN LEFT TO DEFAULT
+    #   f2py picks its backend from the Python version: meson on >=3.12 (where
+    #   distutils is gone), distutils below that. Leaving it implicit means the
+    #   generated service builds through two different toolchains depending on
+    #   the interpreter, which is the opposite of what a re-hosting tool should
+    #   promise — and the distutils path is actively broken, because
+    #   `numpy.distutils` calls a `Compiler.__init__` signature that modern
+    #   setuptools no longer has:
+    #       TypeError: Compiler.__init__() takes from 1 to 3 positional
+    #       arguments but 4 were given
+    #   Reproduced on 3.11 + numpy 2.4 + setuptools 84: the default backend
+    #   exits 1, `--backend meson` exits 0 and the extension computes. CI found
+    #   it on macOS 3.10/3.11 while 3.12 passed, which is exactly the shape of
+    #   a version-dependent backend.
+    #
+    #   numpy.distutils is deprecated and already unavailable on 3.12+, so this
+    #   is the direction of travel regardless. meson + ninja are declared in the
+    #   [build] extra and installed by the generated Dockerfile.
+    return f"""cmake_minimum_required(VERSION 3.18)
+project({service_name} LANGUAGES Fortran)
+
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module NumPy)
+
+execute_process(
+    COMMAND "${{Python_EXECUTABLE}}" -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))"
+    OUTPUT_VARIABLE PY_EXT_SUFFIX
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+set(F2PY_SOURCES
+{sources}
+)
+
+set(F2PY_OUTPUT "${{CMAKE_CURRENT_BINARY_DIR}}/{module.name}${{PY_EXT_SUFFIX}}")
+
+# f2py's meson backend picks tempfile.mkdtemp() for its build directory
+# whenever it isn't told otherwise, and nothing removes that directory or
+# records where it went — orphaning the compile_commands.json and .o files
+# meson always writes there. Pin an explicit, discoverable build directory
+# under the service's own `.nativegate/` tree (the same convention used for
+# `.nativegate/ir.json`) so `ngate build` leaves a real, locatable build
+# tree behind: services/<name>/.nativegate/build/bbdir/compile_commands.json.
+set(F2PY_BUILD_DIR "${{CMAKE_CURRENT_SOURCE_DIR}}/.nativegate/build")
+
+add_custom_command(
+    OUTPUT "${{F2PY_OUTPUT}}"
+    COMMAND "${{Python_EXECUTABLE}}" -m numpy.f2py -c --backend meson --build-dir "${{F2PY_BUILD_DIR}}" -m {module.name} ${{F2PY_SOURCES}}{only_clause}
+    WORKING_DIRECTORY "${{CMAKE_CURRENT_BINARY_DIR}}"
+    DEPENDS ${{F2PY_SOURCES}}
+)
+
+add_custom_target({module.name}_ext ALL DEPENDS "${{F2PY_OUTPUT}}")
+
+install(FILES "${{F2PY_OUTPUT}}" DESTINATION {service_name}/_native)
+"""

nativegate/generators/gateway_gen.py

@@ -0,0 +1,110 @@
+"""Composed-gateway generator: many services, one FastAPI app, one URL.
+
+This is the in-process composition option (design.md section 23, Phase 2/3).
+It trades independent scaling for operational simplicity: one image, one
+deploy, one URL. The alternative — keep every service in its own image and
+put an API gateway / K8s Ingress in front — is Phase 4 and needs no code
+generation at all, just routing config. See docs/deployment-topologies.md.
+"""
+
+from __future__ import annotations
+
+from ..ir import is_valid_python_name
+from . import mcp_gen
+from .error_gen import ERROR_HANDLER_IMPORTS, generate_error_handler
+
+
+def generate_gateway_app(
+    gateway_name: str, service_names: list[str], package_name: str | None = None
+) -> str:
+    for name in service_names:
+        # `from lambda.router import ...` is a SyntaxError, and so is any
+        # service name that is not an identifier — say so here rather than
+        # writing a gateway that cannot be imported.
+        if not is_valid_python_name(name):
+            raise ValueError(
+                f"Service name '{name}' cannot be imported from a gateway: it is "
+                "not a valid Python module name (a keyword, or not an identifier). "
+                "Rename the service."
+            )
+        # A service mounted at /mcp would be shadowed by the gateway's MCP
+        # endpoint, which is mounted at the same path. Starlette resolves the
+        # mount before the router, so the service's routes would simply stop
+        # answering — silently, with no error at startup.
+        if f"/{name}" == mcp_gen.MCP_MOUNT_PATH:
+            raise ValueError(
+                f"Service name '{name}' collides with the gateway's MCP endpoint "
+                f"at {mcp_gen.MCP_MOUNT_PATH}: mounting it would make the service "
+                "unreachable. Rename the service."
+            )
+
+    imports = "\n".join(
+        f"from {name}.router import router as {name}_router" for name in service_names
+    )
+    includes = "\n".join(
+        f'app.include_router({name}_router, prefix="/{name}")' for name in service_names
+    )
+    listed = ", ".join(f'"{name}"' for name in service_names)
+    # The distribution name may be hyphenated ("platform-api"); the logger
+    # and middleware need the importable spelling.
+    package_name = package_name or gateway_name.replace("-", "_")
+    # The mounted routers run under THIS app, so the gateway needs its own copy
+    # of the handler — a handler on a service's own `service.py` app is not
+    # reached in this topology. Same generator, so the two cannot drift.
+    error_imports = "\n".join(ERROR_HANDLER_IMPORTS)
+
+    return f'''# Generated by nativegate. Do not edit by hand — re-run `ngate gateway`.
+"""Composed API gateway: mounts each service's router under its own prefix.
+
+Each mounted service is still built and versioned independently — this app
+only depends on their installed wheels. Swapping to one-image-per-service
+means dropping this file and routing at the ingress instead; the service
+routers are unchanged either way.
+"""
+{error_imports}
+from fastapi import FastAPI, Request
+
+from .middleware import install_middleware, readiness
+{mcp_gen.MCP_APP_IMPORT}
+{imports}
+
+# One MCP endpoint for the whole platform, covering every mounted service —
+# the composed topology's "one URL" promise applies to the LLM interface too.
+# `lifespan=` is required: see generators/mcp_gen.py.
+app = FastAPI(title="{gateway_name}", lifespan=mcp_app.lifespan)
+
+{includes}
+
+# The mounted routers run under THIS app, so the gateway installs its own
+# middleware and readiness probe. Its auth mode is the strictest of the
+# services it mounts — composing an authenticated service into an open gateway
+# would publish it unprotected.
+install_middleware(app, "{package_name}")
+readiness(app, "{package_name}")
+{mcp_gen.mcp_mount_lines()}
+
+
+@app.get("/healthz", tags=["health"])
+def healthz():
+    """Liveness: every mounted service's native extension imported cleanly."""
+    return {{"status": "ok", "services": [{listed}]}}
+{generate_error_handler(gateway_name)}'''
+
+
+def generate_gateway_pyproject(gateway_name: str, package_name: str, service_names: list[str]) -> str:
+    deps = ", ".join(f'"{name}"' for name in service_names)
+    return f'''[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "{gateway_name}"
+version = "1.0.0"
+requires-python = ">=3.10"
+# Each service is a normal wheel dependency — build them independently,
+# publish them, and pin versions here like any other package.
+dependencies = ["fastapi", "uvicorn", "fastmcp", {deps}]
+
+[tool.setuptools]
+packages = ["{package_name}"]
+'''

nativegate/generators/golden_gen.py

@@ -0,0 +1,50 @@
+"""The generated numerical-regression test (see nativegate/golden.py).
+
+Deliberately self-contained: it reads `golden.json` and replays the calls
+recorded there with nothing but the standard library and the service's own
+package. A generated service is a deployable artifact — making its test suite
+import nativegate would mean shipping the code generator alongside the thing
+it generated, or a CI job that passes locally and fails in the container.
+
+The test itself lives in `nativegate/templates/golden_test_template.py` as a
+real, compilable `.py` file rather than a string literal here, so that this
+project's own tooling can see inside it. This module only substitutes the
+three sentinel names in it; see that package's docstring for why substitution
+is textual rather than `str.format`.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+TEMPLATE_PATH = Path(__file__).resolve().parent.parent / "templates" / "golden_test_template.py"
+
+#: Sentinel name in the template -> name of the substitution that replaces it.
+PLACEHOLDERS = {
+    "NATIVEGATE_PACKAGE": "package",
+    "NATIVEGATE_SERVICE_NAME": "service",
+    "NATIVEGATE_GOLDEN_FILENAME": "golden_filename",
+}
+
+_PLACEHOLDER_RE = re.compile(
+    "|".join(sorted(map(re.escape, PLACEHOLDERS), key=len, reverse=True))
+)
+
+
+def _read_template() -> str:
+    return TEMPLATE_PATH.read_text(encoding="utf-8")
+
+
+def generate_golden_test(package_name: str, service_name: str, golden_filename: str) -> str:
+    values = {
+        "package": package_name,
+        "service": service_name,
+        "golden_filename": golden_filename,
+    }
+    # One pass, so a substituted value that happens to contain a sentinel is
+    # never itself rewritten by a later replacement.
+    return _PLACEHOLDER_RE.sub(
+        lambda match: values[PLACEHOLDERS[match.group(0)]],
+        _read_template(),
+    )

nativegate/generators/k8s_gen.py

@@ -0,0 +1,212 @@
+"""Kubernetes manifests for a generated service (design.md sections 4, 20, 21).
+
+`ngate init` created an empty `infrastructure/kubernetes/` and nothing
+else, so every deployment was hand-written from scratch — which in practice
+means the security context and the probes are whatever the person in a hurry
+remembered. This generates them.
+
+The manifests are deliberately *opinionated about safety and vague about
+scale*: replica counts, CPU limits and image registries are yours, but the
+security context, the probe wiring and the COMMON-block worker constraint are
+not negotiable defaults, because each of them is a lesson already paid for
+elsewhere in this codebase.
+
+VALIDATION
+
+`tests/test_k8s_gen.py` parses the output as YAML and asserts on the resulting
+objects, because a substring check passes happily on a file `kubectl` rejects.
+Beyond that, the generated manifests were deserialized into the official
+`kubernetes` client's `V1Deployment` / `V1Service` models and round-tripped:
+every field survived, so nothing here is a name the real API would silently
+drop. That check is not in the suite — it would mean a dependency on the
+Kubernetes client for one test — so re-run it by hand if you add a field.
+
+WHY THE PROBES ARE SPLIT THE WAY THEY ARE
+
+`readinessProbe` -> `/readyz` and `livenessProbe` -> `/healthz`, never the
+other way and never the same endpoint for both:
+
+* `/readyz` reports 503 while the process is draining after SIGTERM. Pointing
+  a *liveness* probe at it would get a draining pod killed mid-request instead
+  of drained — turning a clean rolling deploy into dropped connections.
+* `/healthz` stays up during a drain. Pointing a *readiness* probe at it would
+  keep sending new work to a pod that is shutting down, which is the burst of
+  502s the drain exists to prevent.
+
+WHY `readOnlyRootFilesystem` NEEDS THE emptyDir
+
+A read-only root is worth having — it removes "write a binary and execute it"
+from anything that gets code execution through the native library. But CPython
+wants somewhere to write, and numerical code often wants scratch space, so
+`/tmp` is mounted as an emptyDir. `PYTHONDONTWRITEBYTECODE` is set in the
+image so the interpreter does not try to write `.pyc` next to the source.
+"""
+
+from __future__ import annotations
+
+# Matched to docker_gen: Fortran is pinned to one worker because COMMON blocks
+# are per-process, so a configure call and the compute that reads it must land
+# in the same process. Drifting from that here would silently reintroduce the
+# bug at deploy time, where it is far harder to see.
+_DEFAULT_WORKERS = {"cpp": 2, "fortran": 1}
+
+MANIFEST_FILENAME = "deployment.yaml"
+
+
+def generate_k8s_manifests(
+    service_name: str,
+    language: str,
+    image: str | None = None,
+    *,
+    auth: str = "none",
+    replicas: int = 2,
+) -> str:
+    """A Deployment + Service for one generated service."""
+    image = image or f"{service_name}:latest"
+    workers = _DEFAULT_WORKERS.get(language, 1)
+
+    worker_note = (
+        "        # One worker per POD, not per node: COMMON blocks are\n"
+        "        # per-process. Scale with replicas, never by raising this,\n"
+        "        # unless the exposed routines keep no state between calls.\n"
+        if language == "fortran"
+        else "        # Per-request instances and no process-global state.\n"
+    )
+
+    # Only rendered for api_key services. A secretKeyRef to a Secret that does
+    # not exist keeps the pod in CreateContainerConfigError, which is a
+    # confusing failure to hand someone who never asked for authentication.
+    auth_env = ""
+    auth_note = ""
+    if auth == "api_key":
+        auth_env = f"""
+        - name: NATIVEGATE_API_KEYS
+          valueFrom:
+            secretKeyRef:
+              name: {service_name}-api-keys
+              key: keys"""
+        auth_note = f"""
+#
+# Generated with `api.auth: api_key`. Without keys this service will
+# REFUSE-TO-START, deliberately — see docs/production-readiness.md.
+# Create the Secret before deploying:
+#
+#   kubectl create secret generic {service_name}-api-keys \\
+#     --from-literal=keys="$(openssl rand -hex 32)"
+#"""
+
+    return f"""# Generated by nativegate. Do not edit by hand — re-run `ngate k8s {service_name}`.
+#
+# Replicas, resource limits and the image reference are starting points; the
+# securityContext and the probe wiring are not. See generators/k8s_gen.py for
+# why each is shaped the way it is.{auth_note}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {service_name}
+  labels:
+    app: {service_name}
+spec:
+  replicas: {replicas}
+  selector:
+    matchLabels:
+      app: {service_name}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      # With maxUnavailable 0 a new pod must pass its readiness probe before an
+      # old one is removed, so a native extension that fails to load takes the
+      # rollout down instead of the service.
+      maxUnavailable: 0
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: {service_name}
+    spec:
+      # Long enough for gunicorn's --graceful-timeout 30 to finish in-flight
+      # native calls after /readyz starts reporting 503.
+      terminationGracePeriodSeconds: 45
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        fsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+      - name: {service_name}
+        image: {image}
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: http
+          containerPort: 8000
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop: ["ALL"]
+        env:
+{worker_note}        - name: WEB_CONCURRENCY
+          value: "{workers}"
+        - name: NATIVEGATE_MAX_REQUEST_BYTES
+          value: "1048576"{auth_env}
+        # Draining: reports 503 on SIGTERM so this pod leaves the Service's
+        # endpoints BEFORE it stops accepting. Must not be pointed at /healthz.
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: http
+          periodSeconds: 5
+          timeoutSeconds: 3
+          failureThreshold: 2
+        # Stays up during a drain, so a draining pod is not killed mid-request.
+        # Must not be pointed at /readyz.
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: http
+          periodSeconds: 15
+          timeoutSeconds: 3
+          failureThreshold: 3
+        # A native extension can take a while to load. Without this, a slow
+        # start is read as a failed start and the pod restarts forever.
+        startupProbe:
+          httpGet:
+            path: /healthz
+            port: http
+          periodSeconds: 3
+          failureThreshold: 30
+        resources:
+          requests:
+            cpu: "500m"
+            memory: "256Mi"
+          limits:
+            # No CPU limit on purpose: throttling a native numerical routine
+            # mid-call inflates latency badly and CPU is already bounded by
+            # WEB_CONCURRENCY. Memory IS limited, because a native leak
+            # otherwise takes the node down rather than the pod.
+            memory: "1Gi"
+        volumeMounts:
+        # readOnlyRootFilesystem needs somewhere writable: CPython wants a
+        # temp dir and numerical code often wants scratch space.
+        - name: tmp
+          mountPath: /tmp
+      volumes:
+      - name: tmp
+        emptyDir: {{}}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {service_name}
+  labels:
+    app: {service_name}
+spec:
+  type: ClusterIP
+  selector:
+    app: {service_name}
+  ports:
+  - name: http
+    port: 80
+    targetPort: http
+"""