splent-cli 1.54.0__tar.gz → 1.55.0__tar.gz

{splent_cli-1.54.0/src/splent_cli.egg-info → splent_cli-1.55.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splent_cli
-Version: 1.54.0
+Version: 1.55.0
 Summary: SPLENT-CLI is a CLI to be able to work on your development more easily.
 Author-email: DiversoLab <diversolab@us.es>
 License-Expression: LGPL-3.0-or-later

{splent_cli-1.54.0 → splent_cli-1.55.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "splent_cli"
-version = "1.54.0"
+version = "1.55.0"
 description = "SPLENT-CLI is a CLI to be able to work on your development more easily."
 readme = "README.md"
 requires-python = ">=3.13"

{splent_cli-1.54.0 → splent_cli-1.55.0}/src/splent_cli/commands/feature/feature_pip_install.py

@@ -15,6 +15,13 @@ and because a private feature may deliberately never be published there.
 Without the fallback, one unpublished feature makes the whole product
 unbuildable even though its code is tagged and reachable.
 
+Everything is installed by one pip run. Which channel serves each feature is
+settled first, asking PyPI and the forge for all of them at once, and a tag is
+named by the commit it points at: pip keeps the wheel it builds from a commit
+in its cache, and the next build takes it from there instead of cloning and
+building the feature again. Only when that run fails are the features
+installed one at a time, which says which one failed and why.
+
 Three things about this are worth stating plainly rather than discovering:
 
   * only [tool.splent].features is read. features_dev and features_prod are
@@ -39,7 +46,9 @@ Example:
 """
 
 import os
+import subprocess
 import sys
+from concurrent.futures import ThreadPoolExecutor
 
 import click
 import tomllib
@@ -154,6 +163,180 @@ def _last_line(output: str) -> str:
     return _redacted(lines[-1]) if lines else "no output"
 
 
+# How many PyPI and forge questions are asked at once.
+PROBE_WORKERS = 16
+
+# Seconds one question to PyPI or one ``git ls-remote`` may take.
+PROBE_TIMEOUT = 20
+
+# What a failed pip run is summed up with, counted from its end.
+TAIL_LINES = 15
+
+
+def _asks_pypi() -> bool:
+    """Whether pip installs from PyPI, so PyPI is the one to ask.
+
+    A PIP_INDEX_URL that points elsewhere (a mirror, a private index) serves
+    what PyPI does not say, so nothing is asked and the features are
+    installed one at a time, each trying its index first.
+    """
+    index = os.getenv("PIP_INDEX_URL", "").strip()
+    return not index or index.rstrip("/") in (
+        "https://pypi.org/simple",
+        "https://pypi.python.org/simple",
+    )
+
+
+def _pypi_serves(dist: str, version: str) -> bool | None:
+    """Whether PyPI serves ``dist==version``, None when it could not be asked."""
+    import requests
+
+    url = f"https://pypi.org/pypi/{dist}/{version}/json"
+    for _attempt in range(2):
+        try:
+            response = requests.get(url, timeout=PROBE_TIMEOUT)
+        except requests.RequestException:
+            continue
+        if response.status_code == 200:
+            return True
+        if response.status_code == 404:
+            return False
+    return None
+
+
+def _tag_commit(namespace: str, name: str, ref: str) -> tuple[str, str, str] | None:
+    """(real_url, display_url, commit) of the tag, from the first spelling of
+    the namespace that has it, or None.
+
+    Asked with ``git ls-remote``, so nothing is cloned. An annotated tag is
+    named by the commit it points at, not by the tag object.
+    """
+    env = {
+        **os.environ,
+        **forge.git_auth_env(),
+        "GIT_TERMINAL_PROMPT": "0",
+        "LC_ALL": "C",
+    }
+    for spelling in namespace_spellings(namespace):
+        real, display = https_url(spelling, name)
+        try:
+            result = subprocess.run(
+                ["git", "ls-remote", real, f"refs/tags/{ref}", f"refs/tags/{ref}^{{}}"],
+                capture_output=True,
+                text=True,
+                timeout=PROBE_TIMEOUT,
+                env=env,
+            )
+        except (OSError, subprocess.TimeoutExpired):
+            continue
+        if result.returncode != 0:
+            continue
+        refs = dict(
+            reversed(line.split("\t", 1))
+            for line in result.stdout.splitlines()
+            if "\t" in line
+        )
+        commit = refs.get(f"refs/tags/{ref}^{{}}") or refs.get(f"refs/tags/{ref}")
+        if commit:
+            return real, display, commit
+    return None
+
+
+def _pip_install_many(specs: list[str]) -> tuple[bool, str]:
+    """One pip run for every requirement in ``specs``.
+
+    What pip says is shown as it says it, redacted line by line, so a long
+    build is never silent. Returns whether it succeeded and its last lines.
+    """
+    auth = forge.git_auth_env()
+    cmd = [sys.executable, "-m", "pip", "install", "--progress-bar", "off", *specs]
+    tail: list[str] = []
+    try:
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            env={**os.environ, **auth} if auth else None,
+        )
+    except OSError as exc:
+        return False, str(exc)
+    assert process.stdout is not None
+    for line in process.stdout:
+        line = _redacted(line.rstrip("\n"))
+        click.echo(f"    {line}")
+        tail = [*tail, line][-TAIL_LINES:]
+    return process.wait() == 0, "\n".join(tail)
+
+
+def _requirement(entry, data, pypi_only, asks_pypi):
+    """What one pip run installs for ``entry``.
+
+    Returns (spec, display, from_git), or None when it cannot be settled
+    without trying (PyPI could not be asked, or its tag could not be found),
+    which leaves the entry to the one-at-a-time install.
+    """
+    namespace, name, version, ref = _parse_feature_entry(entry)
+    dist = feature_distribution(data, name)
+    if not version:
+        return dist, dist, None
+    spec = f"{dist}=={version}"
+    if pypi_only:
+        return spec, spec, None
+    if not (ref and namespace):
+        return spec, spec, None
+    if not asks_pypi:
+        return None
+    served = _pypi_serves(dist, version)
+    if served is None:
+        return None
+    if served:
+        return spec, spec, None
+    found = _tag_commit(namespace, name, ref)
+    if found is None:
+        return None
+    real, display, commit = found
+    return (
+        f"{dist} @ git+{real}@{commit}",
+        f"git+{display}@{ref} ({commit[:12]})",
+        f"{name} {ref}",
+    )
+
+
+def _install_together(features, data, pypi_only) -> list[str] | None:
+    """Install every feature in one pip run.
+
+    Returns what came from a git tag, or None when the features have to be
+    installed one at a time: one of them could not be settled in advance, or
+    the run failed and the one-at-a-time install says which feature broke it.
+    """
+    asks_pypi = _asks_pypi()
+    with ThreadPoolExecutor(max_workers=PROBE_WORKERS) as pool:
+        plan = list(
+            pool.map(
+                lambda entry: _requirement(entry, data, pypi_only, asks_pypi),
+                features,
+            )
+        )
+    if any(item is None for item in plan):
+        return None
+
+    for _spec, display, from_git in plan:
+        click.echo(f"  {'git ' if from_git else 'pypi'}  {display}")
+    click.echo()
+    ok, tail = _pip_install_many([spec for spec, _display, _git in plan])
+    click.echo()
+    if not ok:
+        click.secho(
+            "  Installing them together failed. Installing them one at a time, "
+            "to say which one:",
+            fg="yellow",
+        )
+        click.echo()
+        return None
+    return [from_git for _spec, _display, from_git in plan if from_git]
+
+
 @click.command(
     "feature:pip-install",
     short_help="Install the declared features from PyPI, or from their git tag.",
@@ -204,15 +387,22 @@ def feature_pip_install(pypi_only):
 
     click.echo(f"  Installing {len(features)} feature(s)...\n")
 
+    unpinned = [
+        _parse_feature_entry(entry)[1]
+        for entry in features
+        if not _parse_feature_entry(entry)[2]
+    ]
+    together = _install_together(features, data, pypi_only)
+    if together is not None:
+        _summary(features, unpinned, together, [])
+        return
+
     failed = []
-    unpinned = []
     from_git = []
     for entry in features:
         namespace, name, version, ref = _parse_feature_entry(entry)
         dist = feature_distribution(data, name)
         spec = f"{dist}=={version}" if version else dist
-        if not version:
-            unpinned.append(name)
 
         click.echo(f"  installing {spec}")
 
@@ -250,6 +440,11 @@ def feature_pip_install(pypi_only):
             )
             failed.append(spec)
 
+    _summary(features, unpinned, from_git, failed)
+
+
+def _summary(features, unpinned, from_git, failed) -> None:
+    """What was installed from where, and what failed (which exits 1)."""
     click.echo()
     if unpinned:
         click.secho(

{splent_cli-1.54.0 → splent_cli-1.55.0}/src/splent_cli/templates/product/product_Dockerfile.prod.j2

@@ -11,12 +11,20 @@
 # writes those pins, and the two files the image reads, splent.contracts.json
 # and docker/spl/. A clone of the repository builds the same image a release
 # published.
-
-# ── Stage 1: Builder ─────────────────────────────────────────────────────────
+#
+# The image is built in layers that change at different paces, and each is
+# only rebuilt when what it is made from changes: the third-party packages
+# when the lock does, the framework, the CLI and the features when their
+# pins do, the product with every commit. pip keeps what it downloads and
+# builds in a BuildKit cache, so a layer that is rebuilt takes from there
+# what an earlier build already fetched or compiled. A server that updates
+# the image pulls only the layers that changed.
+
+# ── Stage 1: Third-party packages ────────────────────────────────────────────
 # Both stages on the same Debian release, so the packages the builder compiles
 # run on the runtime's libraries. The Python patch release follows the image,
 # which is how its security fixes arrive.
-FROM python:3.13-trixie AS builder
+FROM python:3.13-trixie AS deps
 WORKDIR /workspace
 
 # System dependencies for building
@@ -39,58 +47,70 @@ RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /opt/node/bin/npm \
     && ln -s ../lib/node_modules/npm/bin/npx-cli.js /opt/node/bin/npx
 ENV PATH=/opt/node/bin:$PATH
 
-# The product. Dockerfile.{{ product_name }}.prod.dockerignore keeps the env
-# files (credentials), the development links and the deploy bundle out.
-COPY . {{ product_name }}/
-
-# Same protection for a builder that ignores that file (the classic builder,
-# DOCKER_BUILDKIT=0). Only the runtime stage is pushed, and it copies
-# /workspace from this stage, so the env files never reach a published image.
-RUN find {{ product_name }}/docker -maxdepth 1 -name '.env*' ! -name '*.example' \
-        \( -type f -o -type l \) -delete
-
-# The SPL's UVL model. The framework orders features, and their migrations,
-# by its constraints and looks for it in .splent_cache/spls/. splent
-# product:build keeps the model pyproject.toml pins in docker/spl/, and this
-# puts it where the framework and the CLI look. Without it every boot warns
-# that features load in pyproject.toml order.
-RUN if [ -d {{ product_name }}/docker/spl ]; then \
-        mkdir -p .splent_cache/spls && \
-        cp -R {{ product_name }}/docker/spl/. .splent_cache/spls/; \
-    fi
-
 ENV PIP_ROOT_USER_ACTION=ignore
 
 # Every third-party package at the version docker/requirements.lock pins, as
 # a constraint on every pip install below, so a clone of a release builds the
 # set the release built. splent product:build and product:release write the
 # lock by building with SPLENT_FRESH_LOCK=1, which resolves without it.
+COPY docker/requirements.loc[k] /tmp/lock/
 ARG SPLENT_FRESH_LOCK=""
-RUN if [ -n "$SPLENT_FRESH_LOCK" ] || [ ! -f {{ product_name }}/docker/requirements.lock ]; then \
+RUN if [ -n "$SPLENT_FRESH_LOCK" ] || [ ! -f /tmp/lock/requirements.lock ]; then \
         : > /tmp/constraints.txt; \
     else \
-        cp {{ product_name }}/docker/requirements.lock /tmp/constraints.txt; \
+        cp /tmp/lock/requirements.lock /tmp/constraints.txt; \
     fi
 ENV PIP_CONSTRAINT=/tmp/constraints.txt
 
+# The packages the lock pins, installed before anything SPLENT, so this layer
+# only changes with the lock.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    if grep -q '^[^#[:space:]]' /tmp/constraints.txt; then \
+        pip install -r /tmp/constraints.txt; \
+    fi
+
+# ── Stage 2: Pins ────────────────────────────────────────────────────────────
+# What the builder installs from pyproject.toml, and nothing else of it: the
+# requirements of [tool.splent.runtime] and [project].dependencies, and the
+# features of [tool.splent]. Its version and settings change with every
+# release, and the builder then reinstalls nothing, because these files are
+# the same.
+FROM python:3.13-trixie AS pins
+WORKDIR /pins
+COPY pyproject.toml /tmp/pyproject.toml
+
 # The framework, the CLI and the WSGI server, at the versions
-# [tool.splent.runtime] in pyproject.toml pins (a bare version means exactly
-# that version). splent product:build keeps the framework and the CLI at the
-# versions the product is built with, and a release of the product refuses
-# one PyPI does not serve.
+# [tool.splent.runtime] pins (a bare version means exactly that version),
+# then what the product itself depends on. splent product:build keeps the
+# framework and the CLI at the versions the product is built with, and a
+# release of the product refuses one PyPI does not serve.
 RUN python -c "import sys, tomllib; \
-runtime = tomllib.load(open('{{ product_name }}/pyproject.toml', 'rb')).get('tool', {}).get('splent', {}).get('runtime') \
+data = tomllib.load(open('/tmp/pyproject.toml', 'rb')); \
+runtime = data.get('tool', {}).get('splent', {}).get('runtime') \
     or sys.exit('[tool.splent.runtime] is missing from pyproject.toml. Run splent product:build.'); \
-print('\\n'.join(name + ('==' + str(spec) if str(spec)[:1].isdigit() else str(spec)) for name, spec in runtime.items()))" \
-        > /tmp/runtime.txt \
-    && pip install --no-cache-dir -r /tmp/runtime.txt
-RUN pip install --no-cache-dir ./{{ product_name }}
+print('\\n'.join([name + ('==' + str(spec) if str(spec)[:1].isdigit() else str(spec)) for name, spec in runtime.items()] \
+    + list(data.get('project', {}).get('dependencies', []))))" \
+        > /pins/runtime.txt
+
+# The features, with the PyPI names of those published under one of their
+# own, as feature:pip-install reads them.
+RUN python -c "import json, tomllib; \
+splent = tomllib.load(open('/tmp/pyproject.toml', 'rb')).get('tool', {}).get('splent', {}); \
+lines = ['[tool.splent]'] + [key + ' = ' + json.dumps(splent[key]) for key in ('features', 'features_dev', 'features_prod') if key in splent]; \
+lines += ['[tool.splent.distributions]'] + [json.dumps(k) + ' = ' + json.dumps(v) for k, v in (splent.get('distributions') or {}).items()]; \
+print('\\n'.join(lines))" \
+        > /pins/pyproject.toml
+
+# ── Stage 3: Builder ─────────────────────────────────────────────────────────
+FROM deps AS builder
+
+# Everything SPLENT goes to its own prefix, pip's user site, rather than next
+# to the third-party packages: the runtime stage copies each in its own layer.
+# The product goes to another, since it changes with every commit.
+ENV PYTHONUSERBASE=/opt/splent
+ENV PYTHONPATH=/opt/product
+ENV PATH=/opt/splent/bin:$PATH
 
-# Install the features with pip, from PyPI. This is the ONLY place features are
-# installed in a production image: scripts/00_install_features.sh (pip install -e
-# against the local checkout) belongs to the dev entrypoint and never runs here.
-# A feature declared without a version becomes a bare package name, so pip picks
-# whatever PyPI serves rather than the code in the workspace.
 ENV SPLENT_APP={{ product_name }}
 ENV WORKING_DIR=/workspace
 # The CLI's in-container detection keys on /.dockerenv, which the classic
@@ -112,11 +132,48 @@ ENV SPLENT_FORGE_HOST=${SPLENT_FORGE_HOST}
 ENV SPLENT_FORGE_SSH_HOST=${SPLENT_FORGE_SSH_HOST}
 ENV SPLENT_FORGE_SSH_PORT=${SPLENT_FORGE_SSH_PORT}
 ENV SPLENT_FORGE_API=${SPLENT_FORGE_API}
-RUN splent feature:pip-install
+
+COPY --from=pins /pins/runtime.txt /tmp/runtime.txt
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install --user -r /tmp/runtime.txt
+
+# Install the features with pip, from PyPI. This is the ONLY place features are
+# installed in a production image: scripts/00_install_features.sh (pip install -e
+# against the local checkout) belongs to the dev entrypoint and never runs here.
+# A feature declared without a version becomes a bare package name, so pip picks
+# whatever PyPI serves rather than the code in the workspace.
+COPY --from=pins /pins/pyproject.toml {{ product_name }}/pyproject.toml
+RUN --mount=type=cache,target=/root/.cache/pip \
+    PIP_USER=1 splent feature:pip-install
+
+# The product. Dockerfile.{{ product_name }}.prod.dockerignore keeps the env
+# files (credentials), the development links and the deploy bundle out.
+COPY . {{ product_name }}/
+
+# Same protection for a builder that ignores that file (the classic builder,
+# DOCKER_BUILDKIT=0). Only the runtime stage is pushed, and it copies
+# /workspace from this stage, so the env files never reach a published image.
+RUN find {{ product_name }}/docker -maxdepth 1 -name '.env*' ! -name '*.example' \
+        \( -type f -o -type l \) -delete
+
+# The SPL's UVL model. The framework orders features, and their migrations,
+# by its constraints and looks for it in .splent_cache/spls/. splent
+# product:build keeps the model pyproject.toml pins in docker/spl/, and this
+# puts it where the framework and the CLI look. Without it every boot warns
+# that features load in pyproject.toml order.
+RUN if [ -d {{ product_name }}/docker/spl ]; then \
+        mkdir -p .splent_cache/spls && \
+        cp -R {{ product_name }}/docker/spl/. .splent_cache/spls/; \
+    fi
+
+# What it depends on is already installed, with the runtime.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install --no-deps --target /opt/product ./{{ product_name }}
 
 # The product's own frontend, when its package.json has a build script,
 # installed from its lock exactly as committed.
-RUN cd {{ product_name }} && \
+RUN --mount=type=cache,target=/root/.npm \
+    cd {{ product_name }} && \
     if [ -f package.json ] && [ "$(npm pkg get scripts.build)" != "{}" ]; then \
         test -f package-lock.json || \
             { echo "package-lock.json is missing: run npm install in the product and commit it"; exit 1; }; \
@@ -134,7 +191,7 @@ RUN splent feature:compile --verify
 # The runtime stage copies /workspace, so they would ship otherwise.
 RUN rm -rf {{ product_name }}/node_modules {{ product_name }}/build
 
-# ── Stage 2: Runtime ─────────────────────────────────────────────────────────
+# ── Stage 4: Runtime ─────────────────────────────────────────────────────────
 FROM python:3.13-slim-trixie
 WORKDIR /workspace
 
@@ -150,6 +207,9 @@ ENV PIP_ROOT_USER_ACTION=ignore
 ENV SPLENT_APP={{ product_name }}
 ENV WORKING_DIR=/workspace
 ENV SPLENT_CONTAINER=product
+ENV PYTHONUSERBASE=/opt/splent
+ENV PYTHONPATH=/opt/product
+ENV PATH=/opt/splent/bin:$PATH
 
 # The user the app runs as. The image keeps root as its user so the entrypoint
 # can hand the named volumes to this one before it drops to it: see
@@ -160,9 +220,12 @@ RUN groupadd --system --gid 10001 splent \
     && mkdir -p /var/log/splent \
     && chown splent:splent /var/log/splent
 
-# Copy installed Python packages from builder
-COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages
-COPY --from=builder /usr/local/bin /usr/local/bin
+# The installed packages, one layer each: the third-party ones, then the
+# framework, the CLI and the features, then the product.
+COPY --from=deps /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages
+COPY --from=deps /usr/local/bin /usr/local/bin
+COPY --from=builder /opt/splent /opt/splent
+COPY --from=builder /opt/product /opt/product
 
 # Copy product source (includes compiled assets, templates, migrations,
 # entrypoints), owned by the app user, which writes its state file there.

{splent_cli-1.54.0 → splent_cli-1.55.0/src/splent_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splent_cli
-Version: 1.54.0
+Version: 1.55.0
 Summary: SPLENT-CLI is a CLI to be able to work on your development more easily.
 Author-email: DiversoLab <diversolab@us.es>
 License-Expression: LGPL-3.0-or-later