splent-cli 1.46.0__tar.gz → 1.47.0__tar.gz

{splent_cli-1.46.0/src/splent_cli.egg-info → splent_cli-1.47.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splent_cli
-Version: 1.46.0
+Version: 1.47.0
 Summary: SPLENT-CLI is a CLI to be able to work on your development more easily.
 Author-email: DiversoLab <diversolab@us.es>
 License-Expression: LGPL-3.0-or-later

{splent_cli-1.46.0 → splent_cli-1.47.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "splent_cli"
-version = "1.46.0"
+version = "1.47.0"
 description = "SPLENT-CLI is a CLI to be able to work on your development more easily."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -78,8 +78,10 @@ where = ["src"]
 
 # The templates product:create, feature:create and the rest render. They are
 # data, not modules, so without this a wheel from PyPI shipped none of them.
+# The build context of the frontend toolchain image too, so a CLI installed
+# from PyPI can build the image a feature with frontend/ builds in.
 [tool.setuptools.package-data]
-splent_cli = ["templates/**/*"]
+splent_cli = ["templates/**/*", "toolchain/**/*"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

{splent_cli-1.46.0 → splent_cli-1.47.0}/src/splent_cli/commands/feature/feature_compile.py

@@ -1,8 +1,10 @@
+import dataclasses
 import logging
 import os
 import posixpath
 import shlex
 import subprocess
+from pathlib import Path
 
 import click
 
@@ -368,20 +370,43 @@ def _compile_frontend(root, feature, *, watch, production, release_build) -> boo
 
 
 def _verify_features(features, workspace: str) -> int:
-    """feature:compile --verify: 1 when the manifest of any feature with
-    frontend/ does not match its files, else 0. A pinned feature is checked
-    in the copy the product links, which is what its app serves."""
+    """feature:compile --verify: 1 when the bundle of any feature with
+    frontend/ does not match its manifest, else 0.
+
+    A working copy is checked at the workspace root, a pinned feature in the
+    copy the product links, which is what its app serves, and where neither
+    is, as in the production image, in the package pip installed. When the
+    product has docker/frontend.lock.json (product:build writes it), every
+    feature it lists is checked too, and has to carry the manifest the lock
+    names, of the version it names: the bundle its release built.
+    """
+    from splent_cli.utils.frontend_check import read_lock
+
+    product = context.active_app(workspace)
+    lock = read_lock(os.path.join(workspace, product)) if product else None
+    locked = (lock or {}).get("features") or {}
+    refs = list(features)
+    named = {_split_ref(feature)[1] for feature in refs}
+    for name, entry in sorted(locked.items()):
+        if name not in named:
+            version = entry.get("version")
+            namespace = entry.get("namespace") or "splent_io"
+            refs.append(f"{namespace}/{name}" + (f"@v{version}" if version else ""))
+
     failed = 0
-    for feature in features:
+    for feature in refs:
         namespace, base_name, version = _split_ref(feature)
-        if version:
-            product = context.require_app()
-            root = os.path.join(
-                workspace, product, "features", namespace, f"{base_name}@{version}"
+        entry = locked.get(base_name)
+        copy = _copy_of(workspace, product, namespace, base_name, version)
+        if copy is None and entry is not None:
+            failed += 1
+            click.secho(
+                f"❌ {feature}: docker/frontend.lock.json lists it, and it is "
+                "neither here nor installed",
+                fg="red",
             )
-        else:
-            root = os.path.join(workspace, base_name)
-        if not frontend.declares(root):
+            continue
+        if entry is None and (copy is None or not copy.declares):
             click.echo(
                 click.style(
                     f"    {feature} has no frontend/, nothing to verify.",
@@ -389,7 +414,11 @@ def _verify_features(features, workspace: str) -> int:
                 )
             )
             continue
-        problems = frontend.verify(root, namespace, base_name)
+        problems = frontend.verify_assets(
+            copy.assets, lock=copy.lock, production=bool(version) or copy.installed
+        )
+        if entry is not None:
+            problems += _against_lock(copy, entry, base_name)
         if problems:
             failed += 1
             click.secho(
@@ -402,6 +431,101 @@ def _verify_features(features, workspace: str) -> int:
     return 1 if failed else 0
 
 
+@dataclasses.dataclass
+class _Copy:
+    """The copy of a feature --verify checks: its assets/, its frontend
+    lock (a package pip installed has none), whether pip installed it, and
+    whether it declares frontend/ (an installed one does when it carries a
+    manifest)."""
+
+    assets: Path
+    lock: Path | None
+    installed: bool
+    declares: bool
+
+
+def _copy_of(workspace, product, namespace, base_name, version) -> _Copy | None:
+    """The working copy, the product's link to a pinned release, or else
+    the package pip installed. None when there is none."""
+    if version:
+        root = (
+            os.path.join(
+                workspace, product, "features", namespace, f"{base_name}@{version}"
+            )
+            if product
+            else None
+        )
+    else:
+        root = os.path.join(workspace, base_name)
+    if root and os.path.isdir(root):
+        return _Copy(
+            frontend.assets_dir(root, namespace, base_name),
+            Path(root) / frontend.FRONTEND_DIR / frontend.LOCK,
+            False,
+            frontend.declares(root),
+        )
+    import importlib.util
+
+    try:
+        spec = importlib.util.find_spec(f"{namespace}.{base_name}")
+    except (ImportError, ValueError):
+        spec = None
+    if spec is None or not spec.submodule_search_locations:
+        return None
+    assets = Path(list(spec.submodule_search_locations)[0]) / "assets"
+    return _Copy(assets, None, True, (assets / "meta" / "manifest.json").is_file())
+
+
+def _against_lock(copy: _Copy, entry: dict, base_name: str) -> list[str]:
+    """What does not match between a feature's bundle and its entry in
+    docker/frontend.lock.json."""
+    problems = []
+    wanted = entry.get("manifest_sha256")
+    found = frontend.manifest_sha256(copy.assets)
+    if wanted and found and found != wanted:
+        problems.append(
+            "assets/meta/manifest.json is not the one docker/frontend.lock.json "
+            "names: run splent product:build"
+        )
+    if not copy.installed:
+        return problems
+    import importlib.metadata
+
+    distribution = entry.get("distribution") or base_name
+    try:
+        installed = importlib.metadata.version(distribution)
+    except importlib.metadata.PackageNotFoundError:
+        installed = None
+    if entry.get("version") and installed != entry["version"]:
+        problems.append(
+            f"pip installed {installed or 'no version'}, and "
+            f"docker/frontend.lock.json names {entry['version']}"
+        )
+    if found is None:
+        problems.append(
+            f"it was installed {_channel(distribution)}, without the bundle its "
+            "release built: release it again, so its wheel and its tag carry it"
+        )
+    return problems
+
+
+def _channel(distribution: str) -> str:
+    """How pip installed DISTRIBUTION: from its git tag (git+) or a wheel."""
+    import importlib.metadata
+    import json
+
+    try:
+        direct = importlib.metadata.distribution(distribution).read_text(
+            "direct_url.json"
+        )
+        url = json.loads(direct or "{}")
+    except (importlib.metadata.PackageNotFoundError, ValueError):
+        url = {}
+    if "vcs_info" in url:
+        return f"from its git tag (git+{url.get('url', '')})"
+    return "from a wheel"
+
+
 def _feature_ref(name: str) -> str:
     """*name* as typed, with the ``splent_feature_`` prefix on the feature's
     own name: ``auth`` and ``splent_io/auth`` are ``splent_feature_auth``

{splent_cli-1.46.0 → splent_cli-1.47.0}/src/splent_cli/commands/product/product_build.py

@@ -62,6 +62,10 @@ LEGACY_REQUIREMENTS_FILE = "requirements.prod.txt"
 # Dockerfile applies it as a constraint on every pip install, so two builds of
 # one commit install the same set. Versioned in docker/.
 LOCK_FILE = "requirements.lock"
+# What the features' bundles are and ship, which the image verifies the
+# installed ones against (splent_cli.utils.frontend_check). Versioned in
+# docker/ too.
+FRONTEND_LOCK_FILE = "frontend.lock.json"
 LOCK_HEADER = (
     "# Every third-party package the production image installs, at the version\n"
     "# the last build resolved. Written by splent product:build and\n"
@@ -1202,6 +1206,30 @@ def _contracts_snapshot(workspace: str, declared_features: Sequence[str]) -> dic
     return {"format": CONTRACTS_FORMAT, "features": features}
 
 
+def write_frontend_lock(
+    workspace: str, product_path: str, data: dict, declared_features: Sequence[str]
+) -> str:
+    """Write docker/frontend.lock.json for the features the image installs.
+
+    For each one with frontend/, the release, the sha256 of the manifest its
+    bundle carries and what the bundle ships, with the provider that
+    satisfies each of its peers; the shell; and the features still built in
+    legacy mode (splent_cli.utils.frontend_check.lock). The production
+    image checks every bundle pip installed against it, and the product's
+    CI audits what they ship from it. Returns its path.
+    """
+    from splent_cli.utils import frontend_check
+
+    check = frontend_check.check_product(workspace, product_path, declared_features)
+    path = os.path.join(product_path, "docker", FRONTEND_LOCK_FILE)
+    atomic_write(
+        path,
+        frontend_check.lock_text(frontend_check.lock(check, data)),
+        dir_owner=True,
+    )
+    return path
+
+
 def _not_shipped(directory, names):
     """Bytecode a local run of a feature left behind, never part of the bundle."""
     return {n for n in names if n == "__pycache__" or n.endswith(".pyc")}
@@ -1481,7 +1509,8 @@ def write_deploy_artifacts(
     * ``[tool.splent.runtime]`` in pyproject.toml, the framework and CLI the
       image installs from PyPI;
     * ``splent.contracts.json`` and ``docker/spl/``, the refinement metadata
-      and the UVL model the image reads.
+      and the UVL model the image reads;
+    * ``docker/frontend.lock.json``, the bundles the image verifies.
 
     All of them are versioned with the product. product:build and
     product:release both go through here, so a release commits exactly what
@@ -1791,6 +1820,12 @@ def write_deploy_artifacts(
     written.append(CONTRACTS_SNAPSHOT)
     click.echo(f"✅ Created: {contracts_path}")
 
+    frontend_lock = write_frontend_lock(
+        workspace, product_path, data, declared_features
+    )
+    written.append(f"docker/{FRONTEND_LOCK_FILE}")
+    click.echo(f"✅ Created: {frontend_lock}")
+
     staged_model = stage_spl_model(workspace, product, docker_path)
     if staged_model:
         written.append(f"docker/{SPL_STAGING_DIR}")

splent_cli-1.47.0/src/splent_cli/commands/product/product_frontend_audit.py

@@ -0,0 +1,323 @@
+"""
+product:frontend:audit — what the product's bundles ship, audited as one set.
+
+Each feature with frontend/ audits its own lock when it is released (npm
+audit, services/frontend.py), but a bundle is immutable: an advisory
+published later reaches the product only through a new release of every
+feature that ships the package. This audits the union of what the bundles
+ship, as docker/frontend.lock.json records it (product:build writes it),
+against npm's advisory database, and names the features to release again.
+
+npm audit needs a lock it could install, and the product has none, so the
+packages go straight to the registry's bulk advisory endpoint, the one npm
+audit asks, and each advisory is matched against the shipped version here.
+A critical advisory fails, as it fails a feature's release; a high one is
+said; an audit that cannot reach the registry fails too. --sbom writes the
+CycloneDX SBOM of the whole: the product, each feature and what each one
+ships, with what the features vendor outside npm when their snapshot is at
+hand.
+"""
+
+import datetime
+import json
+import os
+import uuid
+
+import click
+
+from splent_cli.services import compose, context, frontend
+from splent_cli.utils.frontend_check import LOCK_FILE, read_lock
+from splent_cli.utils.npm_semver import satisfies
+
+DEFAULT_REGISTRY = "https://registry.npmjs.org"
+ADVISORIES = "/-/npm/v1/security/advisories/bulk"
+REQUEST_TIMEOUT = 30
+BLOCKING = ("critical",)
+WARNING = ("high",)
+
+
+def shipped(lock: dict) -> dict[str, dict[str, list[str]]]:
+    """``{package: {version: [features]}}`` of what the bundles in LOCK ship."""
+    found: dict[str, dict[str, list[str]]] = {}
+    for name, feature in sorted((lock.get("features") or {}).items()):
+        short = name.removeprefix("splent_feature_")
+        for package, described in (feature.get("bundled") or {}).items():
+            for one in str(described).split(", "):
+                version = one.partition(" ")[0]
+                found.setdefault(package, {}).setdefault(version, []).append(short)
+    return found
+
+
+def registry() -> str:
+    """The npm registry, as npm's own setting names it."""
+    return (os.getenv("NPM_CONFIG_REGISTRY") or DEFAULT_REGISTRY).rstrip("/")
+
+
+def advisories(packages: dict) -> dict:
+    """The registry's advisories for PACKAGES, ``{package: [advisory]}``.
+    Raises click.ClickException when the registry cannot be asked."""
+    import requests
+
+    body = {name: sorted(versions) for name, versions in packages.items()}
+    try:
+        response = requests.post(
+            registry() + ADVISORIES, json=body, timeout=REQUEST_TIMEOUT
+        )
+    except requests.RequestException as e:
+        raise click.ClickException(f"The npm registry could not be asked: {e}")
+    if response.status_code != 200:
+        raise click.ClickException(
+            f"The npm registry answered {response.status_code} to the audit."
+        )
+    try:
+        return response.json()
+    except ValueError:
+        raise click.ClickException("The npm registry's answer is not JSON.")
+
+
+def findings(packages: dict, found: dict) -> list[dict]:
+    """Each advisory of FOUND that covers a version PACKAGES ships, with the
+    features that ship it, most severe first."""
+    order = {"critical": 0, "high": 1, "moderate": 2, "low": 3, "info": 4}
+    result = []
+    for name, versions in packages.items():
+        for advisory in found.get(name) or []:
+            vulnerable = advisory.get("vulnerable_versions") or "*"
+            for version, features in sorted(versions.items()):
+                try:
+                    covered = satisfies(version, vulnerable)
+                except ValueError:
+                    covered = True
+                if covered:
+                    result.append(
+                        {
+                            "package": name,
+                            "version": version,
+                            "features": features,
+                            "severity": advisory.get("severity") or "unknown",
+                            "title": advisory.get("title") or "",
+                            "url": advisory.get("url") or "",
+                        }
+                    )
+    return sorted(
+        result, key=lambda f: (order.get(f["severity"], 5), f["package"], f["version"])
+    )
+
+
+def sbom(lock: dict, workspace: str, product: str, version: str) -> dict:
+    """The CycloneDX 1.5 SBOM of the product's bundles: a component for the
+    product, one for each feature and one for each package a bundle ships,
+    each feature depending on what it ships."""
+    from splent_cli.utils.running_version import cli_version
+
+    components: dict[str, dict] = {}
+    dependencies: list[dict] = []
+    for name, feature in sorted((lock.get("features") or {}).items()):
+        distribution = feature.get("distribution") or name
+        ref = f"pkg:pypi/{distribution}@{feature.get('version') or 'editable'}"
+        components[ref] = {
+            "type": "library",
+            "bom-ref": ref,
+            "name": distribution,
+            "version": feature.get("version") or "editable",
+            "purl": ref,
+            "properties": [
+                {
+                    "name": "splent:manifest_sha256",
+                    "value": feature.get("manifest_sha256"),
+                }
+            ],
+        }
+        depends = []
+        for package, described in sorted((feature.get("bundled") or {}).items()):
+            for one in str(described).split(", "):
+                version_, _, license_ = one.partition(" ")
+                purl = frontend._purl(package, version_)
+                depends.append(purl)
+                components.setdefault(
+                    purl,
+                    {
+                        "type": "library",
+                        "bom-ref": purl,
+                        "name": package,
+                        "version": version_,
+                        "purl": purl,
+                        "licenses": frontend._cyclonedx_licenses(
+                            None if license_ in ("", "UNKNOWN") else license_
+                        ),
+                    },
+                )
+        for vendored in _vendored(workspace, feature, name):
+            depends.append(vendored["bom-ref"])
+            components.setdefault(vendored["bom-ref"], vendored)
+        dependencies.append({"ref": ref, "dependsOn": sorted(set(depends))})
+    product_ref = f"pkg:generic/{product}@{version}"
+    dependencies.insert(
+        0, {"ref": product_ref, "dependsOn": [d["ref"] for d in dependencies]}
+    )
+    return {
+        "bomFormat": "CycloneDX",
+        "specVersion": "1.5",
+        "serialNumber": f"urn:uuid:{uuid.uuid4()}",
+        "version": 1,
+        "metadata": {
+            "timestamp": datetime.datetime.now(datetime.timezone.utc)
+            .replace(microsecond=0)
+            .isoformat(),
+            "tools": {
+                "components": [
+                    {
+                        "type": "application",
+                        "name": "splent_cli",
+                        "version": cli_version(),
+                    }
+                ]
+            },
+            "component": {
+                "type": "application",
+                "bom-ref": product_ref,
+                "name": product,
+                "version": version,
+            },
+        },
+        "components": list(components.values()),
+        "dependencies": dependencies,
+    }
+
+
+def _vendored(workspace: str, feature: dict, name: str) -> list[dict]:
+    """The components a feature's release SBOM lists as vendored (files of
+    a third party that come from no npm package), when its copy is here."""
+    entry = f"{feature.get('namespace') or 'splent_io'}/{name}"
+    if feature.get("version"):
+        entry += f"@v{feature['version']}"
+    root = compose.feature_dir(workspace, entry)
+    assets = frontend.assets_dir(root, feature.get("namespace") or "splent_io", name)
+    try:
+        document = json.loads((assets / "meta" / "sbom.cdx.json").read_text())
+    except (OSError, ValueError):
+        return []
+    return [
+        component
+        for component in document.get("components") or []
+        if {"name": "splent:origin", "value": "vendored"}
+        in (component.get("properties") or [])
+    ]
+
+
+@click.command(
+    "product:frontend:audit",
+    short_help="Audit what the product's bundles ship against npm's advisories.",
+)
+@click.option(
+    "--sbom",
+    "sbom_path",
+    default=None,
+    type=click.Path(dir_okay=False, writable=True),
+    help="Also write the CycloneDX SBOM of the product's bundles to this file.",
+)
+def product_frontend_audit(sbom_path):
+    """
+    Audit the packages the product's bundles ship, as one set.
+
+    \b
+    Reads docker/frontend.lock.json, which product:build writes, and asks
+    npm's registry for the advisories of every package any bundle ships.
+    Each one that covers a shipped version is listed with the features
+    that ship it, which a fix reaches only through a new release of each.
+    A critical advisory fails, a high one is said, the rest are counted.
+    An audit that cannot reach the registry fails too.
+
+    \b
+    --sbom writes the CycloneDX SBOM of the whole: the product, each
+    feature and what it ships.
+
+    \b
+    Examples:
+      splent product:frontend:audit
+      splent product:frontend:audit --sbom frontend.sbom.cdx.json
+    """
+    product = context.require_app()
+    workspace = str(context.workspace())
+    product_dir = os.path.join(workspace, product)
+    lock = read_lock(product_dir)
+    if lock is None:
+        raise click.ClickException(
+            f"{product} has no docker/{LOCK_FILE}. Run splent product:build, which "
+            "writes it."
+        )
+
+    packages = shipped(lock)
+    count = sum(len(versions) for versions in packages.values())
+    click.echo()
+    click.echo(
+        click.style(f"  Auditing what the bundles of {product} ship...", bold=True)
+    )
+    click.echo()
+
+    if sbom_path:
+        document = sbom(lock, workspace, product, _product_version(product_dir))
+        with open(sbom_path, "w", encoding="utf-8") as handle:
+            handle.write(json.dumps(document, indent=2) + "\n")
+        click.secho(
+            f"     ℹ️  SBOM of {len(document['components'])} component(s) written to "
+            f"{sbom_path}",
+            fg="bright_black",
+        )
+
+    failed = False
+    if packages:
+        found = findings(packages, advisories(packages))
+        for item in found:
+            line = (
+                f"{item['package']} {item['version']} ({item['severity']}): "
+                f"{item['title']} {item['url']}".rstrip()
+                + f". Shipped by {', '.join(item['features'])}, to release again "
+                "with a fixed version."
+            )
+            if item["severity"] in BLOCKING:
+                click.secho(f"     ❌ {line}", fg="red")
+                failed = True
+            elif item["severity"] in WARNING:
+                click.secho(f"     ⚠️  {line}", fg="yellow")
+        rest = [f for f in found if f["severity"] not in BLOCKING + WARNING]
+        if rest:
+            click.secho(
+                f"     ℹ️  {len(rest)} advisory(ies) of lower severity, which do not "
+                "fail the audit",
+                fg="bright_black",
+            )
+        if not found:
+            click.secho(
+                f"     ✅ No advisory covers the {count} package version(s) the "
+                f"bundles of {len(lock.get('features') or {})} feature(s) ship",
+                fg="green",
+            )
+    else:
+        click.secho(
+            "     ✅ No bundle of this product ships an npm package", fg="green"
+        )
+
+    click.echo()
+    if failed:
+        click.secho(
+            "  ❌ A bundle ships a package with a critical advisory.",
+            fg="red",
+            bold=True,
+        )
+        click.echo()
+        raise SystemExit(1)
+
+
+def _product_version(product_dir: str) -> str:
+    import tomllib
+
+    try:
+        with open(os.path.join(product_dir, "pyproject.toml"), "rb") as handle:
+            data = tomllib.load(handle)
+    except (OSError, tomllib.TOMLDecodeError):
+        return "unknown"
+    return str(data.get("project", {}).get("version") or "unknown")
+
+
+cli_command = product_frontend_audit

{splent_cli-1.46.0 → splent_cli-1.47.0}/src/splent_cli/commands/product/product_release.py

@@ -11,6 +11,7 @@ import tempfile
 import click
 from splent_cli.commands.product.product_build import (
     CONTRACTS_SNAPSHOT,
+    FRONTEND_LOCK_FILE,
     LOCK_FILE,
     SPL_STAGING_DIR,
     build_production_image,
@@ -126,6 +127,7 @@ RELEASE_ARTIFACTS = (
     # before they exist.
     f"{deploy_bundle.BUNDLE_DIR}/",
     f"docker/{LOCK_FILE}",
+    f"docker/{FRONTEND_LOCK_FILE}",
     CONTRACTS_SNAPSHOT,
     f"docker/{SPL_STAGING_DIR}/",
 )