3lc-compute-plugin-sdk 0.2.2__py3-none-any.whl

tlc_plugin_sdk/shared/model_storage.py

@@ -0,0 +1,160 @@
+# Copyright 2026 3LC Inc.
+# SPDX-License-Identifier: Apache-2.0
+"""Shared utility for saving and copying model checkpoints to Run folders.
+
+Handles both local filesystem and cloud storage (S3, GCS, Azure) via the
+``tlc.Url`` abstraction. The compute service is assumed to have write access
+to the Run folder location.
+"""
+
+from __future__ import annotations
+
+import logging
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def _is_local_url(url_str: str) -> bool:
+    """Check if a URL points to the local filesystem (not cloud storage)."""
+    if "://" not in url_str:
+        return True
+    scheme = url_str.split("://", 1)[0].lower()
+    return scheme in ("file", "")
+
+
+def _resolve_to_local_path(url_str: str) -> Path | None:
+    """Resolve a 3LC URL to a local filesystem path, or None if cloud."""
+    import tlc
+
+    try:
+        abs_url = tlc.Url(url_str).to_absolute()
+        url_s = str(abs_url)
+        if _is_local_url(url_s):
+            # Strip file:// scheme if present
+            path = url_s.replace("file://", "")
+            return Path(path)
+    except Exception:
+        logger.debug("Could not resolve model URL via tlc.Url: %s", url_str)
+    # Try direct path interpretation
+    if _is_local_url(url_str):
+        return Path(url_str)
+    return None
+
+
+def save_model_to_run(
+    run_url: str,
+    model_data: Any,
+    filename: str = "best_model.pt",
+    source_file: str | Path | None = None,
+    on_status: Any = None,
+) -> str:
+    """Save a model checkpoint to a Run's ``model/`` subdirectory.
+
+    Supports both local and cloud (S3/GCS/Azure) run folders. For cloud storage,
+    the file is first written to a temp directory, then uploaded via ``tlc.Url``.
+
+    Args:
+        run_url: The 3LC Run URL (local path or cloud URL).
+        model_data: PyTorch state_dict to save (ignored if ``source_file`` is set).
+        filename: Name for the model file in the run folder.
+        source_file: If set, copy this existing file instead of saving ``model_data``.
+        on_status: Optional callback for status messages.
+
+    Returns:
+        The relative path to the saved model file (e.g. ``model/best.pt``).
+        This is relative to the run folder so it survives run renames.
+
+    Raises:
+        RuntimeError: If the model could not be saved.
+
+    """
+    import tlc
+
+    if on_status is None:
+        on_status = lambda m: None  # noqa: E731
+
+    relative_path = f"model/{filename}"
+
+    local_path = _resolve_to_local_path(run_url)
+
+    if local_path is not None:
+        # ── Local filesystem ──
+        model_dir = local_path / "model"
+        model_dir.mkdir(parents=True, exist_ok=True)
+        dest = model_dir / filename
+
+        if source_file is not None:
+            shutil.copy2(str(source_file), str(dest))
+            on_status(f"Copied model to run: {dest}")
+        else:
+            import torch
+
+            torch.save(model_data, str(dest))
+            on_status(f"Saved model to run: {dest}")
+
+        return relative_path
+
+    # ── Cloud storage (S3/GCS/Azure) ──
+    on_status("Run is on cloud storage, uploading model...")
+    run_abs = tlc.Url(run_url).to_absolute()
+    model_url = run_abs / "model" / filename
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_path = Path(tmp_dir) / filename
+
+        if source_file is not None:
+            shutil.copy2(str(source_file), str(tmp_path))
+        else:
+            import torch
+
+            torch.save(model_data, str(tmp_path))
+
+        # Upload using tlc.Url write facilities
+        try:
+            model_url.write_file(str(tmp_path))
+            on_status(f"Uploaded model to: {model_url}")
+        except AttributeError:
+            # Fallback: use the Url's native copy mechanism
+            src_url = tlc.Url(str(tmp_path))
+            src_url.copy_to(model_url)
+            on_status(f"Copied model to cloud: {model_url}")
+
+    return relative_path
+
+
+def store_model_info_in_run(
+    run: Any,
+    model_name: str,
+    model_path: str,
+    source_url: str = "",
+    on_status: Any = None,
+) -> None:
+    """Store model metadata in a Run's parameters.
+
+    Args:
+        run: The ``tlc.Run`` object.
+        model_name: Model architecture name (e.g. ``yolov8n.pt``, ``resnet50``).
+        model_path: Path/URL to the saved model checkpoint.
+        source_url: Original pretrained model URL (if fine-tuning).
+        on_status: Optional callback for status messages.
+
+    """
+    if on_status is None:
+        on_status = lambda m: None  # noqa: E731
+
+    try:
+        model_info: dict[str, str] = {
+            "model_name": model_name,
+            "model_path": model_path,
+        }
+        if source_url:
+            model_info["model_source"] = source_url
+        run.set_parameters(model_info)
+        on_status(f"Stored model info in Run: {model_name} → {model_path}")
+    except Exception as e:
+        on_status(f"Warning: could not store model info in Run: {e}")
+        logger.warning("Failed to store model info in Run: %s", e)

tlc_plugin_sdk/shared/naming.py

@@ -0,0 +1,84 @@
+# Copyright 2026 3LC Inc.
+# SPDX-License-Identifier: Apache-2.0
+"""Random name generator for runs, jobs, and other entities.
+
+Produces memorable names from data science and computer vision vocabulary.
+"""
+
+from __future__ import annotations
+
+import random
+
+_ADJECTIVES = [
+    "robust",
+    "sparse",
+    "dense",
+    "latent",
+    "frozen",
+    "residual",
+    "pooled",
+    "stacked",
+    "gated",
+    "masked",
+    "dilated",
+    "pretrained",
+    "finetuned",
+    "attentive",
+    "contrastive",
+    "recurrent",
+    "augmented",
+    "calibrated",
+    "distilled",
+    "quantized",
+    "pruned",
+    "anchored",
+    "focal",
+    "semantic",
+    "adaptive",
+    "causal",
+    "spectral",
+    "variational",
+    "embedded",
+    "strided",
+]
+
+_NOUNS = [
+    "gradient",
+    "tensor",
+    "kernel",
+    "encoder",
+    "decoder",
+    "backbone",
+    "embedding",
+    "bottleneck",
+    "convolution",
+    "attention",
+    "dropout",
+    "batchnorm",
+    "activation",
+    "classifier",
+    "regressor",
+    "detector",
+    "segmenter",
+    "feature",
+    "logit",
+    "softmax",
+    "perceptron",
+    "transformer",
+    "diffusion",
+    "manifold",
+    "centroid",
+    "heatmap",
+    "mosaic",
+    "anchor",
+    "pipeline",
+    "checkpoint",
+]
+
+
+def generate_name() -> str:
+    """Generate a memorable name like ``robust-gradient-42``."""
+    adj = random.choice(_ADJECTIVES)
+    noun = random.choice(_NOUNS)
+    num = random.randint(10, 99)
+    return f"{adj}-{noun}-{num}"

tlc_plugin_sdk/shared/ui_inject.py

@@ -0,0 +1,63 @@
+# Copyright 2026 3LC Inc.
+# SPDX-License-Identifier: Apache-2.0
+"""Robustly inject shared client scripts into a plugin UI fragment.
+
+Plugins prepend shared client JS — ``window.PluginJobs`` (job tracking), the alias
+pickers, the config form — into their ``ui.html`` fragment so it runs before the
+fragment's own handlers. The historical ``html.replace("<script>", …, 1)`` idiom
+is a footgun: it matches the first ``<script>`` *substring* anywhere in the file,
+including one mentioned in an HTML comment or a string, and silently injects the
+client into that wrong spot (or nowhere useful) with **no error** — the symptom is
+an ``undefined`` global at click time, far from the cause.
+
+:func:`inject_scripts` targets the first real, inline ``<script>`` *tag* (ignoring
+comments and ``src``-ed scripts) and raises if there is none, so a missing or
+renamed tag fails loudly instead of dropping the client.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Scan for <script ...> opening tags, then exclude any that fall inside an HTML
+# comment so a "<script>" written in the fragment's own docs can't be matched.
+_COMMENT_RE = re.compile(r"<!--.*?-->", re.DOTALL)
+_SCRIPT_OPEN_RE = re.compile(r"<script\b[^>]*>", re.IGNORECASE)
+
+
+def inject_scripts(html: str, *scripts: str) -> str:
+    """Prepend client ``scripts`` inside the fragment's first real ``<script>`` tag.
+
+    Args:
+        html: The plugin UI fragment.
+        scripts: Raw JS bodies (no ``<script>`` tags). Inserted in order, immediately
+            after the opening tag of the first inline ``<script>`` element, so the
+            fragment's own code can rely on them having run.
+
+    Returns:
+        ``html`` with the scripts injected (unchanged if ``scripts`` is empty).
+
+    Raises:
+        ValueError: If ``html`` has no real inline ``<script>`` tag to inject into —
+            loud by design, so a missing/renamed tag can't silently drop the client.
+
+    """
+    if not scripts:
+        return html
+
+    comment_spans = [(m.start(), m.end()) for m in _COMMENT_RE.finditer(html)]
+
+    def _in_comment(pos: int) -> bool:
+        return any(start <= pos < end for start, end in comment_spans)
+
+    for match in _SCRIPT_OPEN_RE.finditer(html):
+        # Skip a <script> mentioned in a comment, and external scripts (a ``src``
+        # script ignores inline content, so injecting into it is a silent no-op).
+        if _in_comment(match.start()) or "src=" in match.group().lower():
+            continue
+        insert_at = match.end()
+        payload = "\n" + "\n".join(scripts) + "\n"
+        return html[:insert_at] + payload + html[insert_at:]
+
+    msg = "inject_scripts: no inline <script> tag found in the fragment to inject into"
+    raise ValueError(msg)

tlc_plugin_sdk/shared/url_utils.py

@@ -0,0 +1,112 @@
+# Copyright 2026 3LC Inc.
+# SPDX-License-Identifier: Apache-2.0
+"""URL normalization utilities for 3LC object URLs.
+
+Ensures file-path URLs are absolute before passing to the tlc SDK,
+preventing the CWD from being prepended to relative-looking paths.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def normalize_url(url: str) -> str:
+    """Normalize a 3LC URL for use with the tlc SDK.
+
+    - If the URL is a protocol URL (e.g. api://, s3://, gs://), return as-is.
+    - If the URL looks like a file path, expand ``~`` and ensure it's absolute.
+    - Handles URL-decoded paths that may have lost their leading slash.
+    """
+    if not url:
+        return url
+
+    # Protocol URLs — pass through
+    if "://" in url:
+        scheme = url.split("://", 1)[0].lower()
+        # File paths on macOS/Linux look like /Users/... not a protocol
+        if scheme in ("api", "s3", "gs", "http", "https", "3lc"):
+            return url
+
+    # File path — expand tilde, then ensure absolute
+    url = os.path.expanduser(url)
+    if not os.path.isabs(url):
+        # Common case: path like "Users/paul/..." that lost its leading /
+        if url.startswith(("Users/", "home/")):
+            return "/" + url
+        # Project-relative URL (e.g. "tinycoco/runs/demo1") — resolve
+        # against the 3LC project root directory.
+        try:
+            import tlc
+
+            project_root = str(tlc.config.project_root_url).rstrip("/")
+            candidate = os.path.join(project_root, url)
+            if os.path.exists(candidate):
+                return candidate
+        except Exception:
+            logger.debug("Could not resolve relative URL against project root: %s", url)
+        # Fallback: return as-is and let the SDK try
+        return url
+
+    return url
+
+
+def normalize_local_path(path: str) -> str:
+    """Normalize a user-typed local filesystem path.
+
+    Strips whitespace and expands ``~``/``~user``. Plugins run with the plugin
+    venv as CWD, so a bare-relative path would silently resolve somewhere no
+    user ever looks — reject it instead.
+
+    Args:
+        path: Raw path string as typed by the user.
+
+    Returns:
+        The expanded, absolute path.
+
+    Raises:
+        ValueError: If the path is empty or not absolute after expansion.
+
+    """
+    expanded = os.path.expanduser(path.strip())
+    if not expanded:
+        msg = "Path is empty."
+        raise ValueError(msg)
+    if not os.path.isabs(expanded):
+        msg = (
+            f"Path must be absolute (got {expanded!r}). "
+            "Relative paths would resolve against the plugin's working directory, not yours."
+        )
+        raise ValueError(msg)
+    return expanded
+
+
+def get_url_column_names(table: Any) -> list[str]:
+    """Return the names of a table's URL/path-valued columns.
+
+    Reads the table's ``_url_columns`` attribute, which is a private 3lc
+    attribute that is not part of the typed public API and may be absent
+    depending on the 3lc version, so reach it defensively. It can be
+    ``[['image']]`` (nested) or ``['image']`` (flat).
+
+    Args:
+        table: A ``tlc.Table``.
+
+    Returns:
+        Flat list of column names; empty if none could be determined.
+
+    """
+    names: list[str] = []
+    try:
+        for entry in list(getattr(table, "_url_columns", [])):
+            if isinstance(entry, list):
+                names.extend(str(col) for col in entry)
+            else:
+                names.append(str(entry))
+    except Exception:
+        logger.debug("Could not extract URL column names from table", exc_info=True)
+    return names