nextmv 1.7.4.dev5__tar.gz → 1.8.0.dev1__tar.gz

{nextmv-1.7.4.dev5 → nextmv-1.8.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextmv
-Version: 1.7.4.dev5
+Version: 1.8.0.dev1
 Summary: The all-purpose Python SDK for Nextmv
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextmv-py.docs.nextmv.io/en/latest/nextmv/

nextmv-1.8.0.dev1/nextmv/__about__.py

@@ -0,0 +1 @@
+__version__ = "v1.8.0.dev1"

nextmv-1.8.0.dev1/nextmv/cache.py

@@ -0,0 +1,431 @@
+"""
+Module with the cache interface and operations for working with dependencies.
+
+Functions
+---------
+get_cache
+    Get general information about the cache, including number of dependencies and total size.
+clear_cache
+    Delete the entire cache directory and recreate it empty.
+dep_cache_key
+    Return a SHA-256 hex digest that uniquely identifies a single package.
+get_cached_dep
+    Return the path to the cached per-package `installed.tar.gz` on a hit.
+store_dep
+    Store a per-package `installed.tar.gz` into the cache and run LRU eviction.
+format_bytes
+    Return a human-friendly string representation of a byte count.
+"""
+
+import hashlib
+import json
+import os
+import re
+import shutil
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+_CACHE_DIR = Path.home() / ".nextmv" / "cache"
+"""Root directory for all Nextmv cache data."""
+_DEPS_CACHE_DIR = _CACHE_DIR / "deps"
+"""Directory storing per-package dependency tarballs."""
+_CACHE_INFO_FILE = "cache_info.json"
+"""Name of the metadata file stored alongside each cached entry."""
+_DEP_TARBALL_NAME = "installed.tar.gz"
+"""Name of the per-package tarball containing installed files for a single package."""
+_MAX_DEPS = 2000
+"""Maximum number of individual package tarballs to keep in the cache."""
+_MAX_DEP_BYTES = 5 * 1024**3
+"""Maximum total cache size in bytes (5 GiB)."""
+
+
+def get_cache() -> dict[str, Any]:
+    """
+    Gets general information about the cache.
+
+    Returns
+    -------
+    dict[str, Any]
+        A dictionary containing information about the cache, such as the number of
+        dependencies cached and the total size of the cache in bytes.
+    """
+
+    num_deps, total_bytes = _cache_stats()
+
+    dependencies = []
+    if _DEPS_CACHE_DIR.is_dir():
+        for entry_dir in _DEPS_CACHE_DIR.iterdir():
+            if not entry_dir.is_dir():
+                continue
+
+            info_file = entry_dir / _CACHE_INFO_FILE
+            try:
+                with open(info_file) as f:
+                    info = json.load(f)
+
+                tarball = entry_dir / _DEP_TARBALL_NAME
+                try:
+                    tarball_size = format_bytes(tarball.stat().st_size)
+                except OSError:
+                    tarball_size = ""
+
+                dependencies.append(
+                    {
+                        "key": entry_dir.name,
+                        "name": info.get("name", ""),
+                        "version": info.get("version", ""),
+                        "last_used_at": info.get("last_used_at", ""),
+                        "python_version": info.get("python_version", ""),
+                        "platform": info.get("platform", ""),
+                        "size": tarball_size,
+                    }
+                )
+            except Exception:
+                pass
+
+    if dependencies:
+        dependencies.sort(key=lambda d: (d["last_used_at"], d["name"], d["version"]), reverse=True)
+
+    return {
+        "num_dependencies": num_deps,
+        "total_size": format_bytes(total_bytes),
+        "dependencies": dependencies,
+    }
+
+
+def clear_cache() -> tuple[int, int]:
+    """
+    Delete the entire cache directory and recreate it empty.
+
+    Returns
+    -------
+    tuple[int, int]
+        A tuple of (num_deps_deleted, total_bytes_deleted).
+    """
+
+    if not _CACHE_DIR.exists():
+        return 0, 0
+
+    num_deps, total_bytes = _cache_stats()
+    shutil.rmtree(str(_CACHE_DIR))
+    _create_cache()
+
+    return num_deps, total_bytes
+
+
+def dep_cache_key(name: str, version: str, python_version: str, platform: str) -> str:
+    """
+    Return a SHA-256 hex digest that uniquely identifies a single package wheel.
+
+    The key is derived from the normalised package name, version, target Python
+    version, and platform.  Name normalisation follows PEP 503 so that
+    `pydantic-core` and `pydantic_core` map to the same key.
+
+    Parameters
+    ----------
+    name : str
+        The package name as it appears in the lockfile or wheel filename.
+    version : str
+        The pinned version string (e.g. `"2.23.4"`).
+    python_version : str
+        The target Python version string (e.g. `"3.11"`).
+    platform : str
+        The target platform string (e.g. `"aarch64-unknown-linux-gnu"`).
+
+    Returns
+    -------
+    str
+        A 64-character lowercase hex SHA-256 digest.
+    """
+
+    normalized = re.sub(r"[-_.]+", "_", name.lower())
+    raw = f"{normalized}=={version}|{python_version}|{platform}"
+
+    return hashlib.sha256(raw.encode()).hexdigest()
+
+
+def get_cached_dep(pkg_key: str) -> Path | None:
+    """
+    Return the path to the cached `installed.tar.gz` for *pkg_key*, or `None`
+    on a miss.
+
+    The tarball contains the installed files for a single package archived under
+    `.nextmv/python/deps/`, ready to be concatenated with other per-package
+    tarballs to form the final `deps.tar.gz` (valid per RFC 1952).
+
+    On a cache hit `last_used_at` in the entry's `cache_info.json` is
+    updated to the current UTC time for LRU tracking.
+
+    Parameters
+    ----------
+    pkg_key : str
+        The cache key returned by :func:`dep_cache_key`.
+
+    Returns
+    -------
+    Path or None
+        Path to the `installed.tar.gz` file, or `None` when no entry exists.
+    """
+
+    entry_dir = _DEPS_CACHE_DIR / pkg_key
+    info_file = entry_dir / _CACHE_INFO_FILE
+    installed_tar = entry_dir / _DEP_TARBALL_NAME
+
+    if not installed_tar.is_file() or not info_file.is_file():
+        return None
+
+    # Update last_used_at for true LRU tracking.
+    try:
+        with open(info_file) as f:
+            info = json.load(f)
+
+        info["last_used_at"] = _now_iso()
+        _write_json_atomic(path=info_file, data=info)
+    except Exception:
+        pass
+
+    return installed_tar
+
+
+def store_dep(
+    pkg_key: str,
+    tar_path: Path,
+    name: str,
+    version: str,
+    python_version: str,
+    platform: str,
+    max_entries: int = _MAX_DEPS,
+    max_bytes: int = _MAX_DEP_BYTES,
+) -> None:
+    """
+    Store a per-package `installed.tar.gz` in the cache and run LRU eviction.
+
+    The tarball must contain the installed files for a single package archived
+    under `.nextmv/python/deps/`.  The copy is performed atomically via a
+    sibling temporary directory followed by a rename.
+
+    Parameters
+    ----------
+    pkg_key : str
+        The cache key returned by :func:`dep_cache_key`.
+    tar_path : Path
+        Path to the `installed.tar.gz` to cache.
+    name : str
+        The package name (e.g. `"pydantic-core"`).
+    version : str
+        The pinned version string (e.g. `"2.23.4"`).
+    python_version : str
+        The target Python version string (e.g. `"3.11"`).
+    platform : str
+        The target platform string (e.g. `"aarch64-unknown-linux-gnu"`).
+    max_entries : int, optional
+        Maximum number of package tarball entries to retain.
+    max_bytes : int, optional
+        Maximum total cache size in bytes.
+    """
+
+    _create_cache()
+    entry_dir = _DEPS_CACHE_DIR / pkg_key
+
+    with tempfile.TemporaryDirectory(dir=_DEPS_CACHE_DIR, prefix=f"{pkg_key}-tmp-", ignore_cleanup_errors=True) as _tmp:
+        tmp_entry = Path(_tmp)
+        shutil.copy2(str(tar_path), tmp_entry / _DEP_TARBALL_NAME)
+        now = _now_iso()
+        info = {
+            "created_at": now,
+            "last_used_at": now,
+            "name": name,
+            "version": version,
+            "python_version": python_version,
+            "platform": platform,
+        }
+        _write_json_atomic(path=tmp_entry / _CACHE_INFO_FILE, data=info)
+
+        # Rename into place; if the key already exists (race), keep existing.
+        if not entry_dir.exists():
+            tmp_entry.rename(entry_dir)
+
+    _evict_lru(max_entries=max_entries, max_bytes=max_bytes)
+
+
+def format_bytes(size: int) -> str:
+    """
+    Return a human-friendly string representation of a byte count.
+
+    Parameters
+    ----------
+    size : int
+        The size in bytes to format.
+
+    Returns
+    -------
+    str
+        A human-friendly string using B, KiB, MiB, or GiB units.
+    """
+
+    if size < 1024:
+        return f"{size} B"
+    elif size < 1024 * 1024:
+        return f"{size / 1024:.2f} KiB"
+    elif size < 1024 * 1024 * 1024:
+        return f"{size / (1024 * 1024):.2f} MiB"
+    else:
+        return f"{size / (1024 * 1024 * 1024):.2f} GiB"
+
+
+def _cache_stats() -> tuple[int, int]:
+    """
+    Return the number of cached dependency entries and their total size in bytes.
+
+    Returns
+    -------
+    tuple[int, int]
+        A tuple of (num_deps, total_bytes).
+    """
+
+    num_deps = 0
+    total_bytes = 0
+
+    if not _DEPS_CACHE_DIR.is_dir():
+        return num_deps, total_bytes
+
+    for entry_dir in _DEPS_CACHE_DIR.iterdir():
+        if entry_dir.is_dir():
+            num_deps += 1
+            total_bytes += _dir_size(entry_dir)
+
+    return num_deps, total_bytes
+
+
+def _create_cache() -> None:
+    """
+    Create the cache directories if they do not already exist.
+    """
+
+    _DEPS_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Remove leftover staging dirs from previous crashed/killed runs.
+    for entry in _DEPS_CACHE_DIR.iterdir():
+        if entry.is_dir() and "-tmp-" in entry.name:
+            shutil.rmtree(str(entry), ignore_errors=True)
+
+
+def _evict_lru(max_entries: int = _MAX_DEPS, max_bytes: int = _MAX_DEP_BYTES) -> int:
+    """
+    Remove the least-recently-used per-package tarball cache entries until both
+    caps are satisfied.
+
+    Parameters
+    ----------
+    max_entries : int, optional
+        Maximum number of package tarball entries to retain.
+    max_bytes : int, optional
+        Maximum total cache size in bytes.
+
+    Returns
+    -------
+    int
+        The number of entries that were removed.
+    """
+
+    if not _DEPS_CACHE_DIR.is_dir():
+        return 0
+
+    entries = []
+    for entry_dir in _DEPS_CACHE_DIR.iterdir():
+        if not entry_dir.is_dir() or "-tmp-" in entry_dir.name:
+            continue
+
+        info_file = entry_dir / _CACHE_INFO_FILE
+        # If the info file has problems, treat the entry as very old.
+        try:
+            with open(info_file) as f:
+                info = json.load(f)
+
+            last_used = datetime.fromisoformat(info["last_used_at"])
+        except Exception:
+            last_used = datetime.min.replace(tzinfo=timezone.utc)
+
+        entries.append(
+            {
+                "last_used": last_used,
+                "dir": entry_dir,
+                "size": _dir_size(entry_dir),
+            }
+        )
+
+    entries.sort(key=lambda e: e["last_used"])
+
+    total = sum(e["size"] for e in entries)
+    evicted = 0
+    while entries and (len(entries) > max_entries or total > max_bytes):
+        oldest = entries.pop(0)
+        shutil.rmtree(str(oldest["dir"]), ignore_errors=True)
+        total -= oldest["size"]
+        evicted += 1
+
+    return evicted
+
+
+def _now_iso() -> str:
+    """
+    Return the current UTC time as an ISO-8601 string.
+
+    Returns
+    -------
+    str
+        Current UTC time as an ISO-8601 string.
+    """
+
+    return datetime.now(tz=timezone.utc).isoformat()
+
+
+def _dir_size(path: Path) -> int:
+    """
+    Return the total size in bytes of all files under path.
+
+    Parameters
+    ----------
+    path : Path
+        The directory path to calculate the size of.
+
+    Returns
+    -------
+    int
+        Total size in bytes of all files under path.
+    """
+
+    total = 0
+    for root, _, files in os.walk(path):
+        for fname in files:
+            try:
+                total += os.path.getsize(os.path.join(root, fname))
+            except OSError:
+                pass
+
+    return total
+
+
+def _write_json_atomic(path: Path, data: dict) -> None:
+    """
+    Write data as JSON to path atomically via a sibling temporary file.
+
+    Parameters
+    ----------
+    path : Path
+        The path to write the JSON data to.
+    data : dict
+        The data to write as JSON.
+    """
+
+    with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, prefix=".tmp-", delete=False) as f:
+        json.dump(data, f, indent=2)
+        tmp = Path(f.name)
+
+    try:
+        os.replace(tmp, path)
+    except Exception as e:
+        tmp.unlink(missing_ok=True)
+        raise e

nextmv-1.8.0.dev1/nextmv/cli/cache/__init__.py

@@ -0,0 +1,21 @@
+"""
+This module defines the cache command tree for the Nextmv CLI.
+"""
+
+import typer
+
+from nextmv.cli.cache.delete import app as delete_app
+from nextmv.cli.cache.get import app as get_app
+
+# Set up subcommand application.
+app = typer.Typer()
+app.add_typer(delete_app)
+app.add_typer(get_app)
+
+
+@app.callback()
+def callback() -> None:
+    """
+    Manage the cache used for Nextmv operations.
+    """
+    pass

nextmv-1.8.0.dev1/nextmv/cli/cache/delete.py

@@ -0,0 +1,48 @@
+"""
+This module defines the cache delete command for the Nextmv CLI.
+"""
+
+import typer
+
+from nextmv.cache import clear_cache, format_bytes
+from nextmv.cli.message import confirmation, info, success
+from nextmv.cli.options import YesOption
+
+# Set up subcommand application.
+app = typer.Typer()
+
+
+@app.command()
+def delete(yes: YesOption = False) -> None:
+    """
+    Deletes the Nextmv cache and resets it to an empty state.
+
+    This action is permanent and cannot be undone. Use the --yes flag to skip
+    the confirmation prompt.
+
+    [bold][underline]Examples[/underline][/bold]
+
+    - Delete the Nextmv cache.
+        $ [dim]nextmv cache delete[/dim]
+
+    - Delete the Nextmv cache without confirmation prompt.
+        $ [dim]nextmv cache delete --yes[/dim]
+    """
+
+    deps, bytes_deleted = clear_cache()
+    if deps == 0 and bytes_deleted == 0:
+        info("Nothing to delete: cache is already empty.")
+        return
+
+    if not yes:
+        confirm = confirmation(
+            "Are you sure you want to delete the Nextmv cache? This action cannot be undone.",
+        )
+
+        if not confirm:
+            info("The Nextmv cache will not be deleted.")
+            return
+
+    size_str = format_bytes(bytes_deleted)
+    dep_label = "dependency" if deps == 1 else "dependencies"
+    success(f"Deleted [magenta]{deps}[/magenta] {dep_label} ([magenta]{size_str}[/magenta]) from the cache.")

nextmv-1.8.0.dev1/nextmv/cli/cache/get.py

@@ -0,0 +1,27 @@
+"""
+This module defines the cache get command for the Nextmv CLI.
+"""
+
+import typer
+
+from nextmv.cache import get_cache
+from nextmv.cli.message import print_json, success
+
+# Set up subcommand application.
+app = typer.Typer()
+
+
+@app.command()
+def get() -> None:
+    """
+    Gets general information about the Nextmv cache.
+
+    [bold][underline]Examples[/underline][/bold]
+
+    - Get cache information.
+        $ [dim]nextmv cache get[/dim]
+    """
+
+    info = get_cache()
+    success("Cache information retrieved successfully.")
+    print_json(info)

{nextmv-1.7.4.dev5 → nextmv-1.8.0.dev1}/nextmv/cli/cloud/app/push.py

@@ -40,6 +40,13 @@ def push(
             metavar="MANIFEST_PATH",
         ),
     ] = None,
+    no_cache: Annotated[
+        bool,
+        typer.Option(
+            "--no-cache",
+            help="Do not read from or write to the Nextmv dependency cache for [magenta]Python[/magenta] applications.",
+        ),
+    ] = False,
     # Options for version control.
     version_id: Annotated[
         str | None,
@@ -184,6 +191,7 @@ def push(
         update_instance_id=update_instance_id,
         create_defined=create_defined,
         create_instance_id=create_instance_id,
+        no_cache=no_cache,
     )
 
 
@@ -199,6 +207,7 @@ def handle_push(
     update_instance_id: str | None,
     create_defined: bool,
     create_instance_id: str | None,
+    no_cache: bool = False,
 ) -> str:
     """
     Handle the core push flow: push the application, create a version, and link it to an instance.
@@ -227,6 +236,9 @@ def handle_push(
         Whether --create-instance-id was provided.
     create_instance_id : str | None
         The instance ID to create.
+    no_cache : bool
+        Whether to disable the dependency cache when packaging Python
+        applications.
 
     Returns
     -------
@@ -242,6 +254,7 @@ def handle_push(
         app_dir=app_dir,
         verbose=True,
         rich_print=True,
+        no_cache=no_cache,
     )
 
     now = datetime.now(timezone.utc)

{nextmv-1.7.4.dev5 → nextmv-1.8.0.dev1}/nextmv/cli/main.py

@@ -23,6 +23,7 @@ import rich
 import typer
 from typer import rich_utils
 
+from nextmv.cli.cache import app as cache_app
 from nextmv.cli.cloud import app as cloud_app
 from nextmv.cli.community import app as community_app
 from nextmv.cli.configuration import app as configuration_app
@@ -52,6 +53,7 @@ app = typer.Typer(
 
 # Register subcommands. The `name` parameter is required when the subcommand
 # module has a callback function defined.
+app.add_typer(cache_app, name="cache")
 app.add_typer(cloud_app, name="cloud")
 app.add_typer(community_app, name="community")
 app.add_typer(configuration_app, name="configuration")

{nextmv-1.7.4.dev5 → nextmv-1.8.0.dev1}/nextmv/cloud/application/__init__.py

@@ -414,7 +414,7 @@ class Application(
             # Re-throw the exception if it is not the expected 404 error.
             raise e from None
 
-    def push(  # noqa: C901
+    def push(
         self,
         manifest: Manifest | None = None,
         app_dir: str | None = None,
@@ -422,6 +422,7 @@ class Application(
         model: Model | None = None,
         model_configuration: ModelConfiguration | None = None,
         rich_print: bool = False,
+        no_cache: bool = False,
     ) -> None:
         """
         Push an app to Nextmv Cloud.
@@ -457,6 +458,13 @@ class Application(
             with `model`.
         rich_print : bool, default=False
             Whether to use rich printing when verbose output is enabled.
+        no_cache : bool, default=False
+            When working with Python, dependencies are cached to speed up
+            subsequent pushes. Setting no_cache to True will skip using the
+            cache and force a fresh build of all dependencies. This is useful
+            when you want to ensure that you are pushing the most up-to-date
+            versions of your dependencies, or if you are encountering issues
+            with the cache and want to rule it out as a potential cause.
 
         Raises
         ------
@@ -550,9 +558,17 @@ class Application(
         if (model is None and model_configuration is not None) or (model is not None and model_configuration is None):
             raise ValueError("model and model_configuration must be provided together")
 
-        package._run_build_command(app_dir, manifest.build, verbose, rich_print)
-        package._run_pre_push_command(app_dir, manifest.pre_push, verbose, rich_print)
-        tar_file, output_dir = package._package(app_dir, manifest, model, model_configuration, verbose, rich_print)
+        package.run_build_command(app_dir, manifest.build, verbose, rich_print)
+        package.run_pre_push_command(app_dir, manifest.pre_push, verbose, rich_print)
+        tar_file, output_dir = package.package(
+            app_dir,
+            manifest,
+            model,
+            model_configuration,
+            verbose,
+            rich_print,
+            no_cache,
+        )
         self.__update_app_binary(tar_file, manifest, verbose, rich_print)
 
         try: