lamin_cli 1.1.0__tar.gz → 1.3.0__tar.gz

lamin_cli-1.3.0/.github/workflows/build.yml

@@ -0,0 +1,82 @@
+name: build
+
+on:
+  push:
+    branches: [release]
+  pull_request:
+
+jobs:
+  pre-filter:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: dorny/paths-filter@v3
+        id: changes
+        if: github.event_name != 'push'
+        with:
+          filters: |
+            modal:
+              - 'lamin_cli/compute/modal.py'
+              - 'tests/modal/**'
+
+      - id: set-matrix
+        shell: bash
+        run: |
+          BASE_GROUPS=$(jq -n -c '[]')
+
+          if [[ "${{ github.event_name }}" == "push" || "${{ steps.changes.outputs.modal }}" == "true" ]]; then
+            # Run everything on push or when modal paths change
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: ($groups + ["modal"])}')
+          else
+            # Otherwise only run base groups
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: $groups}')
+          fi
+
+          # Output as single line for GitHub Actions
+          echo "matrix=$(echo "$MATRIX" | jq -c .)" >> $GITHUB_OUTPUT
+
+          # Pretty print for debugging
+          echo "Generated matrix:"
+          echo "$MATRIX" | jq .
+
+  test:
+    needs: pre-filter
+    runs-on: ubuntu-latest
+    env:
+      LAMIN_API_KEY: ${{ secrets.LAMIN_API_KEY_TESTUSER1 }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.pre-filter.outputs.matrix)}}
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: cache pre-commit
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
+
+      - run: pip install git+https://github.com/laminlabs/laminci
+
+      - run: uv pip install --system modal pytest
+
+      - run: modal token set --token-id ${{ secrets.MODAL_DEV_TOKEN_ID }} --token-secret ${{ secrets.MODAL_DEV_TOKEN_SECRET }}
+
+      - run: nox -s setup
+
+      - run: lamin login
+
+      - run: pytest tests/modal

{lamin_cli-1.1.0 → lamin_cli-1.3.0}/.gitignore

@@ -1,4 +1,5 @@
 # LaminDB
+modal_mount_dir/
 lamindb_docs/
 _build
 mydata/

{lamin_cli-1.1.0 → lamin_cli-1.3.0}/.pre-commit-config.yaml

@@ -16,8 +16,8 @@ repos:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --unsafe-fixes]
       - id: ruff-format
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v4.0.0-alpha.8
+  - repo: https://github.com/rbubley/mirrors-prettier
+    rev: v3.5.1
     hooks:
       - id: prettier
   - repo: https://github.com/kynan/nbstripout

{lamin_cli-1.1.0 → lamin_cli-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamin_cli
-Version: 1.1.0
+Version: 1.3.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

lamin_cli-1.3.0/lamin_cli/__init__.py

@@ -0,0 +1,3 @@
+"""Lamin CLI."""
+
+__version__ = "1.3.0"

{lamin_cli-1.1.0 → lamin_cli-1.3.0}/lamin_cli/__main__.py

@@ -9,6 +9,13 @@ from functools import wraps
 from importlib.metadata import PackageNotFoundError, version
 from typing import TYPE_CHECKING
 
+from lamindb_setup._init_instance import (
+    DOC_DB,
+    DOC_INSTANCE_NAME,
+    DOC_MODULES,
+    DOC_STORAGE_ARG,
+)
+
 if TYPE_CHECKING:
     from collections.abc import Mapping
 
@@ -41,12 +48,7 @@ else:
         "lamin": [
             {
                 "name": "Connect to an instance",
-                "commands": [
-                    "connect",
-                    "disconnect",
-                    "info",
-                    "init",
-                ],
+                "commands": ["connect", "disconnect", "info", "init", "run"],
             },
             {
                 "name": "Read & write data",
@@ -155,24 +157,20 @@ def schema_to_modules_callback(ctx, param, value):
 
 # fmt: off
 @main.command()
-@click.option("--storage", type=str, help="Local directory, s3://bucket_name, gs://bucket_name.")
-@click.option("--db", type=str, default=None, help="Postgres database connection URL, do not pass for SQLite.")
-@click.option("--modules", type=str, default=None, help="Comma-separated string of schema modules.")
-@click.option("--name", type=str, default=None, help="The instance name.")
-@click.option("--schema", type=str, default=None, help="[DEPRECATED] Use --modules instead.", callback=schema_to_modules_callback)
+@click.option("--storage", type=str, default = ".", help=DOC_STORAGE_ARG)
+@click.option("--name", type=str, default=None, help=DOC_INSTANCE_NAME)
+@click.option("--db", type=str, default=None, help=DOC_DB)
+@click.option("--modules", type=str, default=None, help=DOC_MODULES)
 # fmt: on
 def init(
     storage: str,
+    name: str | None,
     db: str | None,
     modules: str | None,
-    name: str | None,
-    schema: str | None,
 ):
     """Init an instance."""
     from lamindb_setup._init_instance import init as init_
 
-    modules = modules if modules is not None else schema
-
     return init_(storage=storage, db=db, modules=modules, name=name)
 
 
@@ -188,6 +186,8 @@ def connect(instance: str):
     `lamin connect` switches
     {attr}`~lamindb.setup.core.SetupSettings.auto_connect` to `True` so that you
     auto-connect in a Python session upon importing `lamindb`.
+
+    For manually connecting in a Python session, use {func}`~lamindb.connect`.
     """
     from lamindb_setup import connect as connect_
     from lamindb_setup import settings as settings_
@@ -279,45 +279,103 @@ def load(entity: str, uid: str | None = None, key: str | None = None, with_env:
 @click.argument("entity", type=str)
 @click.option("--uid", help="The uid for the entity.")
 @click.option("--key", help="The key for the entity.")
-@click.option(
-    "--with-env", is_flag=True, help="Also return the environment for a tranform."
-)
-def get(entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False):
+def get(entity: str, uid: str | None = None, key: str | None = None):
     """Query metadata about an entity.
 
-    Currently only works for artifact & transform and behaves like `lamin load`.
+    Currently only works for artifact.
     """
-    from lamin_cli._load import load as load_
+    import lamindb_setup as ln_setup
+
+    from ._load import decompose_url
+
+    if entity.startswith("https://") and "lamin" in entity:
+        url = entity
+        instance, entity, uid = decompose_url(url)
+    elif entity not in {"artifact"}:
+        raise SystemExit("Entity has to be a laminhub URL or 'artifact'")
+    else:
+        instance = ln_setup.settings.instance.slug
+
+    ln_setup.connect(instance)
+    import lamindb as ln
 
-    click.echo(f"! to load a file or folder, please use: lamin load {entity}")
-    return load_(entity, uid=uid, key=key, with_env=with_env)
+    if uid is not None:
+        artifact = ln.Artifact.get(uid)
+    else:
+        artifact = ln.Artifact.get(key=key)
+    artifact.describe()
 
 
 @main.command()
-@click.argument("filepath", type=click.Path(exists=True, dir_okay=True, file_okay=True))
+@click.argument("path", type=click.Path(exists=True, dir_okay=True, file_okay=True))
 @click.option("--key", type=str, default=None)
 @click.option("--description", type=str, default=None)
+@click.option("--stem-uid", type=str, default=None)
 @click.option("--registry", type=str, default=None)
-def save(filepath: str, key: str, description: str, registry: str):
+def save(path: str, key: str, description: str, stem_uid: str, registry: str):
     """Save a file or folder.
 
-    Defaults to saving `.py` and `.ipynb` as {class}`~lamindb.Transform` and
-    other file types and folders as {class}`~lamindb.Artifact`.
-
-    You can save a `.py` or `.ipynb` file as an {class}`~lamindb.Artifact` by
-    passing `--registry artifact`.
+    Defaults to saving `.py`, `.ipynb`, `.R`, `.Rmd`, and `.qmd` as {class}`~lamindb.Transform` and
+    other file types and folders as {class}`~lamindb.Artifact`. You can save a `.py` or `.ipynb` file as
+    an {class}`~lamindb.Artifact` by passing `--registry artifact`.
     """
     from lamin_cli._save import save_from_filepath_cli
 
-    if save_from_filepath_cli(filepath, key, description, registry) is not None:
+    if save_from_filepath_cli(path, key, description, stem_uid, registry) is not None:
         sys.exit(1)
 
 
+@main.command()
+@click.argument("filepath", type=str)
+@click.option("--project", type=str, default=None, help="A valid project name or uid. When running on Modal, creates an app with the same name.", required=True)
+@click.option("--image-url", type=str, default=None, help="A URL to the base docker image to use.")
+@click.option("--packages", type=str, default="lamindb", help="A comma-separated list of additional packages to install.")
+@click.option("--cpu", type=float, default=None, help="Configuration for the CPU.")
+@click.option("--gpu", type=str, default=None, help="The type of GPU to use (only compatible with cuda images).")
+def run(filepath: str, project: str, image_url: str, packages: str, cpu: int, gpu: str | None):
+    """Run a compute job in the cloud.
+
+    This is an EXPERIMENTAL feature that enables to run a script on Modal.
+
+    Example: Given a valid project name "my_project".
+
+    ```
+    lamin run my_script.py --project my_project
+    ```
+    """
+    import shutil
+    from pathlib import Path
+
+    from lamin_cli.compute.modal import Runner
+
+    default_mount_dir = Path('./modal_mount_dir')
+    if not default_mount_dir.is_dir():
+        default_mount_dir.mkdir(parents=True, exist_ok=True)
+
+    shutil.copy(filepath, default_mount_dir)
+
+    filepath_in_mount_dir = Path(default_mount_dir) / Path(filepath).name
+
+    package_list = []
+    if packages:
+        package_list = [package.strip() for package in packages.split(',')]
+
+    runner = Runner(
+        local_mount_dir=default_mount_dir,
+        app_name=project,
+        packages=package_list,
+        image_url=image_url,
+        cpu=cpu,
+        gpu=gpu
+    )
+
+    runner.run(filepath_in_mount_dir)
+
+
 main.add_command(settings)
 main.add_command(cache)
 main.add_command(migrate)
 
-
 # https://stackoverflow.com/questions/57810659/automatically-generate-all-help-documentation-for-click-commands
 # https://claude.ai/chat/73c28487-bec3-4073-8110-50d1a2dd6b84
 def _generate_help():

lamin_cli-1.3.0/lamin_cli/_load.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import re
+import shutil
+from pathlib import Path
+
+from lamin_utils import logger
+
+
+def decompose_url(url: str) -> tuple[str, str, str]:
+    assert any(keyword in url for keyword in ["transform", "artifact", "collection"])
+    for entity in ["transform", "artifact", "collection"]:
+        if entity in url:
+            break
+    uid = url.split(f"{entity}/")[1]
+    instance_slug = "/".join(url.split("/")[3:5])
+    return instance_slug, entity, uid
+
+
+def load(
+    entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False
+):
+    import lamindb_setup as ln_setup
+
+    if entity.startswith("https://") and "lamin" in entity:
+        url = entity
+        instance, entity, uid = decompose_url(url)
+    elif entity not in {"artifact", "transform", "collection"}:
+        raise SystemExit(
+            "Entity has to be a laminhub URL or 'artifact', 'collection', or 'transform'"
+        )
+    else:
+        instance = ln_setup.settings.instance.slug
+
+    ln_setup.connect(instance)
+    import lamindb as ln
+
+    def script_to_notebook(
+        transform: ln.Transform, notebook_path: Path, bump_revision: bool = False
+    ) -> None:
+        import jupytext
+        from lamin_utils._base62 import increment_base62
+
+        if notebook_path.suffix == ".ipynb":
+            # below is backward compat
+            if "# # transform.name" in transform.source_code:
+                new_content = transform.source_code.replace(
+                    "# # transform.name", f"# # {transform.description}"
+                )
+            elif transform.source_code.startswith("# %% [markdown]"):
+                source_code_split = transform.source_code.split("\n")
+                if source_code_split[1] == "#":
+                    source_code_split[1] = f"# # {transform.description}"
+                new_content = "\n".join(source_code_split)
+            else:
+                new_content = transform.source_code
+        else:  # R notebook
+            # Pattern to match title only within YAML header section
+            title_pattern = r'^---\n.*?title:\s*"([^"]*)".*?---'
+            title_match = re.search(
+                title_pattern, transform.source_code, flags=re.DOTALL | re.MULTILINE
+            )
+            new_content = transform.source_code
+            if title_match:
+                current_title = title_match.group(1)
+                if current_title != transform.description:
+                    pattern = r'^(---\n.*?title:\s*)"([^"]*)"(.*?---)'
+                    replacement = f'\\1"{transform.description}"\\3'
+                    new_content = re.sub(
+                        pattern,
+                        replacement,
+                        new_content,
+                        flags=re.DOTALL | re.MULTILINE,
+                    )
+                    logger.important(
+                        f"updated title to match description: {current_title} →"
+                        f" {transform.description}"
+                    )
+        if bump_revision:
+            uid = transform.uid
+            if uid in new_content:
+                new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
+                new_content = new_content.replace(uid, new_uid)
+                logger.important(f"updated uid: {uid} → {new_uid}")
+        if notebook_path.suffix == ".ipynb":
+            notebook = jupytext.reads(new_content, fmt="py:percent")
+            jupytext.write(notebook, notebook_path)
+        else:
+            notebook_path.write_text(new_content)
+
+    query_by_uid = uid is not None
+
+    match entity:
+        case "transform":
+            if query_by_uid:
+                # we don't use .get here because DoesNotExist is hard to catch
+                # due to private django API
+                # here full uid is not expected anymore as before
+                # via ln.Transform.objects.get(uid=uid)
+                transforms = ln.Transform.objects.filter(uid__startswith=uid)
+            else:
+                # if below, we take is_latest=True as the criterion, we might get draft notebooks
+                # hence, we use source_code__isnull=False and order by created_at instead
+                transforms = ln.Transform.objects.filter(
+                    key=key, source_code__isnull=False
+                )
+
+            if (n_transforms := len(transforms)) == 0:
+                err_msg = f"uid {uid}" if query_by_uid else f"key={key} and source_code"
+                raise SystemExit(f"Transform with {err_msg} does not exist.")
+
+            if n_transforms > 1:
+                transforms = transforms.order_by("-created_at")
+            transform = transforms.first()
+
+            target_relpath = Path(transform.key)
+            if len(target_relpath.parents) > 1:
+                logger.important(
+                    "preserve the folder structure for versioning:"
+                    f" {target_relpath.parent}/"
+                )
+                target_relpath.parent.mkdir(parents=True, exist_ok=True)
+            if target_relpath.exists():
+                response = input(f"! {target_relpath} exists: replace? (y/n)")
+                if response != "y":
+                    raise SystemExit("Aborted.")
+
+            if transform.source_code is not None:
+                if target_relpath.suffix in (".ipynb", ".Rmd", ".qmd"):
+                    script_to_notebook(transform, target_relpath, bump_revision=True)
+                else:
+                    target_relpath.write_text(transform.source_code)
+            else:
+                raise SystemExit("No source code available for this transform.")
+
+            logger.important(f"{transform.type} is here: {target_relpath}")
+
+            if with_env:
+                ln.settings.track_run_inputs = False
+                if (
+                    transform.latest_run is not None
+                    and transform.latest_run.environment is not None
+                ):
+                    filepath_env_cache = transform.latest_run.environment.cache()
+                    target_env_filename = (
+                        target_relpath.parent
+                        / f"{target_relpath.stem}__requirements.txt"
+                    )
+                    shutil.move(filepath_env_cache, target_env_filename)
+                    logger.important(f"environment is here: {target_env_filename}")
+                else:
+                    logger.warning(
+                        "latest transform run with environment doesn't exist"
+                    )
+
+            return target_relpath
+        case "artifact" | "collection":
+            ln.settings.track_run_inputs = False
+
+            EntityClass = ln.Artifact if entity == "artifact" else ln.Collection
+
+            # we don't use .get here because DoesNotExist is hard to catch
+            # due to private django API
+            if query_by_uid:
+                entities = EntityClass.filter(uid__startswith=uid)
+            else:
+                entities = EntityClass.filter(key=key)
+
+            if (n_entities := len(entities)) == 0:
+                err_msg = f"uid={uid}" if query_by_uid else f"key={key}"
+                raise SystemExit(
+                    f"{entity.capitalize()} with {err_msg} does not exist."
+                )
+
+            if n_entities > 1:
+                entities = entities.order_by("-created_at")
+
+            entity_obj = entities.first()
+            cache_path = entity_obj.cache()
+
+            # collection gives us a list of paths
+            if isinstance(cache_path, list):
+                logger.important(f"{entity} paths ({len(cache_path)} files):")
+                for i, path in enumerate(cache_path):
+                    if i < 5 or i >= len(cache_path) - 5:
+                        logger.important(f"  [{i + 1}/{len(cache_path)}] {path}")
+                    elif i == 5:
+                        logger.important(f"  ... {len(cache_path) - 10} more files ...")
+            else:
+                logger.important(f"{entity} is here: {cache_path}")
+        case _:
+            raise AssertionError(f"unknown entity {entity}")

{lamin_cli-1.1.0 → lamin_cli-1.3.0}/lamin_cli/_save.py

@@ -46,6 +46,7 @@ def save_from_filepath_cli(
     filepath: str | Path,
     key: str | None,
     description: str | None,
+    stem_uid: str | None,
     registry: str | None,
 ) -> str | None:
     import lamindb_setup as ln_setup
@@ -99,10 +100,21 @@ def save_from_filepath_cli(
 
     if registry == "artifact":
         ln.settings.creation.artifact_silence_missing_run_warning = True
-        if key is None and description is None:
+        revises = None
+        if stem_uid is not None:
+            revises = (
+                ln.Artifact.filter(uid__startswith=stem_uid)
+                .order_by("-created_at")
+                .first()
+            )
+            if revises is None:
+                raise ln.errors.InvalidArgument("The stem uid is not found.")
+        elif key is None and description is None:
             logger.error("Please pass a key or description via --key or --description")
             return "missing-key-or-description"
-        artifact = ln.Artifact(filepath, key=key, description=description).save()
+        artifact = ln.Artifact(
+            filepath, key=key, description=description, revises=revises
+        ).save()
         logger.important(f"saved: {artifact}")
         logger.important(f"storage path: {artifact.path}")
         if ln_setup.settings.storage.type == "s3":
@@ -125,6 +137,15 @@ def save_from_filepath_cli(
                 )
                 return "not-tracked-in-transform-registry"
         else:
+            revises = None
+            if stem_uid is not None:
+                revises = (
+                    ln.Transform.filter(uid__startswith=stem_uid)
+                    .order_by("-created_at")
+                    .first()
+                )
+                if revises is None:
+                    raise ln.errors.InvalidArgument("The stem uid is not found.")
             # TODO: build in the logic that queries for relative file paths
             # we have in Context; add tests for multiple versions
             transform = ln.Transform.filter(
@@ -135,6 +156,7 @@ def save_from_filepath_cli(
                     description=filepath.name,
                     key=filepath.name,
                     type="script" if filepath.suffix in {".R", ".py"} else "notebook",
+                    revises=revises,
                 ).save()
                 logger.important(f"created Transform('{transform.uid}')")
         # latest run of this transform by user
@@ -147,6 +169,11 @@ def save_from_filepath_cli(
             )
             if response != "y":
                 return "aborted-save-notebook-created-by-different-user"
+        if run is None and transform.key.endswith(".ipynb"):
+            run = ln.Run(transform=transform).save()
+            logger.important(
+                f"found no run, creating Run('{run.uid}') to display the html"
+            )
         return_code = save_context_core(
             run=run,
             transform=transform,

lamin_cli-1.3.0/lamin_cli/compute/modal.py

@@ -0,0 +1,175 @@
+import os
+import subprocess
+import sys
+import threading
+from pathlib import Path
+
+import lamindb_setup as ln_setup
+import modal
+
+
+def run_script(path: Path) -> dict:
+    """Takes a path to a script for running it as a function through Modal."""
+    result = {"success": False, "output": "", "error": ""}
+
+    def stream_output(stream, capture_list):
+        """Read from stream line by line and print in real-time while also capturing to a list."""
+        for line in iter(stream.readline, ""):
+            print(line, end="")  # Print in real-time
+            capture_list.append(line)
+        stream.close()
+
+    if not path.exists():
+        raise FileNotFoundError(f"Script file not found: {path}")
+
+    try:
+        # Run the script using subprocess
+        process = subprocess.Popen(
+            [sys.executable, path.as_posix()],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,  # Line buffered
+        )
+
+        # Capture output and error while streaming stdout in real-time
+        stdout_lines: list[str] = []
+        stderr_lines: list[str] = []
+
+        # Create threads to handle stdout and stderr streams
+        stdout_thread = threading.Thread(
+            target=stream_output, args=(process.stdout, stdout_lines)
+        )
+        stderr_thread = threading.Thread(
+            target=stream_output, args=(process.stderr, stderr_lines)
+        )
+
+        # Set as daemon threads so they exit when the main program exits
+        stdout_thread.daemon = True
+        stderr_thread.daemon = True
+
+        # Start the threads
+        stdout_thread.start()
+        stderr_thread.start()
+
+        # Wait for the process to complete
+        return_code = process.wait()
+
+        # Wait for the threads to finish
+        stdout_thread.join()
+        stderr_thread.join()
+
+        # Join the captured output
+        stdout_output = "".join(stdout_lines)
+        stderr_output = "".join(stderr_lines)
+
+        # Check return code
+        if return_code == 0:
+            result["success"] = True
+            result["output"] = stdout_output
+        else:
+            result["error"] = stderr_output
+
+    except Exception as e:
+        import traceback
+
+        result["error"] = str(e) + "\n" + traceback.format_exc()
+    return result
+
+
+class Runner:
+    def __init__(
+        self,
+        app_name: str,
+        local_mount_dir: str | Path = "./scripts",
+        remote_mount_dir: str | Path = "/scripts",
+        image_url: str | None = None,
+        packages: list[str] | None = None,
+        cpu: float | None = None,
+        gpu: str | None = None,
+    ):
+        self.app_name = app_name  # we use the LaminDB project name as the app name
+        self.app = self.create_modal_app(app_name)
+
+        self.local_mount_dir = local_mount_dir
+        self.remote_mount_dir = remote_mount_dir
+
+        self.image = self.create_modal_image(
+            local_dir=local_mount_dir, packages=packages, image_url=image_url
+        )
+
+        local_secrets = self._configure_local_secrets()
+
+        self.modal_function = self.app.function(
+            image=self.image, cpu=cpu, gpu=gpu, secrets=[local_secrets]
+        )(run_script)
+
+    def run(self, script_local_path: Path) -> None:
+        script_remote_path = self.local_to_remote_path(str(script_local_path))
+        with modal.enable_output(show_progress=True):  # Prints out modal logs
+            with self.app.run():
+                self.modal_function.remote(Path(script_remote_path))
+
+    def create_modal_app(self, app_name: str) -> modal.App:
+        app = modal.App(app_name)
+        return app
+
+    def local_to_remote_path(self, local_path: str | Path) -> str:
+        local_path = Path(local_path).absolute()
+        local_mount_dir = Path(self.local_mount_dir).absolute()
+        remote_mount_dir = Path(self.remote_mount_dir)
+
+        # Check if local_path is inside local_mount_dir
+        try:
+            # This will raise ValueError if local_path is not relative to local_mount_dir
+            relative_path = local_path.relative_to(local_mount_dir)
+        except ValueError as err:
+            raise ValueError(
+                f"Local path '{local_path}' is not inside the mount directory '{local_mount_dir}'"
+            ) from err
+
+        # Join remote_mount_dir with the relative path
+        remote_path = remote_mount_dir / relative_path
+
+        # Return as string with normalized separators
+        return remote_path.as_posix()
+
+    def _configure_local_secrets(self) -> dict:
+        if ln_setup.settings.user.api_key is None:
+            raise ValueError("Please authenticate via: lamin login")
+
+        all_env_variables = {
+            "LAMIN_API_KEY": ln_setup.settings.user.api_key,
+            "LAMIN_CURRENT_PROJECT": self.app_name,
+            "LAMIN_CURRENT_INSTANCE": ln_setup.settings.instance.slug,
+        }
+        local_secrets = modal.Secret.from_dict(all_env_variables)
+        return local_secrets
+
+    def create_modal_image(
+        self,
+        python_version: str = "3.12",
+        packages: list[str] | None = None,
+        local_dir: str | Path = "./scripts",
+        remote_dir: str = "/scripts/",
+        image_url: str | None = None,
+        env_variables: dict | None = None,
+    ) -> modal.Image:
+        if env_variables is None:
+            env_variables = {}
+        if packages is None:
+            packages = ["lamindb"]
+        else:
+            packages.append("lamindb")  # Append lamindb to the list of packages
+
+        if image_url is None:
+            image = modal.Image.debian_slim(python_version=python_version)
+        else:
+            image = modal.Image.from_registry(image_url, add_python=python_version)
+        return (
+            image.pip_install(packages)
+            .env(env_variables)
+            .add_local_python_source("lamindb", "lamindb_setup", copy=True)
+            .run_commands("lamin settings set auto-connect true")
+            .add_local_dir(local_dir, remote_dir)
+        )

lamin_cli-1.3.0/noxfile.py

@@ -0,0 +1,13 @@
+import os
+
+import nox
+from laminci.nox import install_lamindb
+
+IS_PR = os.getenv("GITHUB_EVENT_NAME") != "push"
+nox.options.default_venv_backend = "none"
+
+
+@nox.session
+def setup(session):
+    branch = "main" if IS_PR else "release"
+    install_lamindb(session, branch=branch)

{lamin_cli-1.1.0/tests → lamin_cli-1.3.0/tests/core}/test_load.py

@@ -60,7 +60,15 @@ def test_load_transform():
     path2.unlink()
 
 
-def test_load_artifact():
+def test_get_load_artifact():
+    result = subprocess.run(
+        "lamin get"
+        " 'https://lamin.ai/laminlabs/lamin-site-assets/artifact/e2G7k9EVul4JbfsEYAy5'",
+        shell=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0
+
     result = subprocess.run(
         "lamin load"
         " 'https://lamin.ai/laminlabs/lamin-site-assets/artifact/e2G7k9EVul4JbfsEYAy5'",
@@ -76,3 +84,12 @@ def test_load_artifact():
         capture_output=True,
     )
     assert result.returncode == 0
+
+
+def test_load_collection():
+    result = subprocess.run(
+        "lamin load 'https://lamin.ai/laminlabs/lamindata/collection/2wUs6V1OuGzp5Ll4'",
+        shell=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0

{lamin_cli-1.1.0/tests → lamin_cli-1.3.0/tests/core}/test_multi_process.py

@@ -2,7 +2,7 @@ import subprocess
 from multiprocessing import Process
 from pathlib import Path
 
-scripts_dir = Path(__file__).parent.resolve() / "scripts"
+scripts_dir = Path(__file__).parent.parent.resolve() / "scripts"
 
 
 def run_script():

{lamin_cli-1.1.0/tests → lamin_cli-1.3.0/tests/core}/test_save_files.py

@@ -3,7 +3,7 @@ from pathlib import Path
 
 import lamindb_setup as ln_setup
 
-test_file = Path(__file__).parent.parent.resolve() / ".gitignore"
+test_file = Path(__file__).parent.parent.parent.resolve() / ".gitignore"
 
 
 def test_save_file():
@@ -44,7 +44,7 @@ def test_save_file():
     )
     print(result.stdout.decode())
     print(result.stderr.decode())
-    assert "found artifact with same hash" in result.stdout.decode()
+    assert "returning existing artifact with same hash" in result.stdout.decode()
     assert "key='mytest'" in result.stdout.decode()
     assert "storage path:" in result.stdout.decode()
     assert result.returncode == 0

{lamin_cli-1.1.0/tests → lamin_cli-1.3.0/tests/core}/test_save_notebooks.py

@@ -184,9 +184,7 @@ print("my consecutive cell")
     new_path = notebook_path.with_name("new_name.ipynb")
     os.system(f"cp {notebook_path} {new_path}")
 
-    # upon re-running it, the user is asked to create a new stem uid
-    with pytest.raises(CellExecutionError) as error:
-        nbproject_test.execute_notebooks(new_path, print_outputs=True)
-
-    print(error.exconly())
-    assert "clashes with the existing key" in error.exconly()
+    # upon re-running it, the notebook name is updated
+    nbproject_test.execute_notebooks(new_path, print_outputs=True)
+    transform = ln.Transform.get("hlsFXswrJjtt0001")
+    assert "new_name.ipynb" in transform.key

{lamin_cli-1.1.0/tests → lamin_cli-1.3.0/tests/core}/test_save_r_code.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 import lamindb as ln
 
-scripts_dir = Path(__file__).parent.resolve() / "scripts"
+scripts_dir = Path(__file__).parent.parent.resolve() / "scripts"
 
 
 def test_run_save_cache():

{lamin_cli-1.1.0/tests → lamin_cli-1.3.0/tests/core}/test_save_scripts.py

@@ -5,7 +5,7 @@ from pathlib import Path
 import lamindb as ln
 from lamindb_setup import settings
 
-scripts_dir = Path(__file__).parent.resolve() / "scripts"
+scripts_dir = Path(__file__).parent.parent.resolve() / "scripts"
 
 
 def test_save_without_uid():

lamin_cli-1.3.0/tests/modal/test_modal.py

@@ -0,0 +1,19 @@
+import subprocess
+from pathlib import Path
+
+scripts_dir = Path(__file__).parent.parent.resolve() / "scripts"
+
+
+def test_run_on_modal():
+    filepath = scripts_dir / "run-track-and-finish.py"
+
+    subprocess.run("lamin connect laminlabs/lamindata", shell=True, check=True)
+    result = subprocess.run(
+        f"lamin run {filepath} --project 1QLbS6N7wwiL",
+        shell=True,
+        capture_output=True,
+    )
+    print(result.stdout.decode())
+    assert result.returncode == 0
+    assert "hello!" in result.stdout.decode()
+    assert "finished Run" in result.stdout.decode()

{lamin_cli-1.1.0 → lamin_cli-1.3.0}/tests/scripts/run-track-and-finish.py

@@ -1,9 +1,5 @@
 import lamindb as ln
 
-# purposefully do not pass uid
-ln.track()
-
-
 if __name__ == "__main__":
     # we're using new_run here to mock the notebook situation
     # and cover the look up of an existing run in the tests

lamin_cli-1.1.0/lamin_cli/__init__.py

@@ -1,3 +0,0 @@
-"""Lamin CLI."""
-
-__version__ = "1.1.0"

lamin_cli-1.1.0/lamin_cli/_load.py

@@ -1,169 +0,0 @@
-from __future__ import annotations
-
-import re
-import shutil
-from pathlib import Path
-
-from lamin_utils import logger
-
-
-def decompose_url(url: str) -> tuple[str, str, str]:
-    assert "transform" in url or "artifact" in url
-    for entity in ["transform", "artifact"]:
-        if entity in url:
-            break
-    uid = url.split(f"{entity}/")[1]
-    instance_slug = "/".join(url.split("/")[3:5])
-    return instance_slug, entity, uid
-
-
-def load(
-    entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False
-):
-    import lamindb_setup as ln_setup
-
-    if entity.startswith("https://") and "lamin" in entity:
-        url = entity
-        instance, entity, uid = decompose_url(url)
-    elif entity not in {"artifact", "transform"}:
-        raise SystemExit("Entity has to be a laminhub URL or 'artifact' or 'transform'")
-    else:
-        instance = ln_setup.settings.instance.slug
-
-    ln_setup.connect(instance)
-    import lamindb as ln
-
-    def script_to_notebook(
-        transform: ln.Transform, notebook_path: Path, bump_revision: bool = False
-    ) -> None:
-        import jupytext
-        from lamin_utils._base62 import increment_base62
-
-        if notebook_path.suffix == ".ipynb":
-            # below is backward compat
-            if "# # transform.name" in transform.source_code:
-                new_content = transform.source_code.replace(
-                    "# # transform.name", f"# # {transform.description}"
-                )
-            elif transform.source_code.startswith("# %% [markdown]"):
-                source_code_split = transform.source_code.split("\n")
-                if source_code_split[1] == "#":
-                    source_code_split[1] = f"# # {transform.description}"
-                new_content = "\n".join(source_code_split)
-            else:
-                new_content = transform.source_code
-        else:  # R notebook
-            # Pattern to match title only within YAML header section
-            title_pattern = r'^---\n.*?title:\s*"([^"]*)".*?---'
-            title_match = re.search(
-                title_pattern, transform.source_code, flags=re.DOTALL | re.MULTILINE
-            )
-            new_content = transform.source_code
-            if title_match:
-                current_title = title_match.group(1)
-                if current_title != transform.description:
-                    pattern = r'^(---\n.*?title:\s*)"([^"]*)"(.*?---)'
-                    replacement = f'\\1"{transform.description}"\\3'
-                    new_content = re.sub(
-                        pattern,
-                        replacement,
-                        new_content,
-                        flags=re.DOTALL | re.MULTILINE,
-                    )
-                    logger.important(
-                        f"updated title to match description: {current_title} →"
-                        f" {transform.description}"
-                    )
-        if bump_revision:
-            uid = transform.uid
-            if uid in new_content:
-                new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
-                new_content = new_content.replace(uid, new_uid)
-                logger.important(f"updated uid: {uid} → {new_uid}")
-        if notebook_path.suffix == ".ipynb":
-            notebook = jupytext.reads(new_content, fmt="py:percent")
-            jupytext.write(notebook, notebook_path)
-        else:
-            notebook_path.write_text(new_content)
-
-    query_by_uid = uid is not None
-
-    if entity == "transform":
-        if query_by_uid:
-            # we don't use .get here because DoesNotExist is hard to catch
-            # due to private django API
-            # here full uid is not expected anymore as before
-            # via ln.Transform.objects.get(uid=uid)
-            transforms = ln.Transform.objects.filter(uid__startswith=uid)
-        else:
-            # if below, we take is_latest=True as the criterion, we might get draft notebooks
-            # hence, we use source_code__isnull=False and order by created_at instead
-            transforms = ln.Transform.objects.filter(key=key, source_code__isnull=False)
-
-        if (n_transforms := len(transforms)) == 0:
-            err_msg = f"uid {uid}" if query_by_uid else f"key={key} and source_code"
-            raise SystemExit(f"Transform with {err_msg} does not exist.")
-
-        if n_transforms > 1:
-            transforms = transforms.order_by("-created_at")
-        transform = transforms.first()
-
-        target_relpath = Path(transform.key)
-        if len(target_relpath.parents) > 1:
-            logger.important(
-                "preserve the folder structure for versioning:"
-                f" {target_relpath.parent}/"
-            )
-            target_relpath.parent.mkdir(parents=True, exist_ok=True)
-        if target_relpath.exists():
-            response = input(f"! {target_relpath} exists: replace? (y/n)")
-            if response != "y":
-                raise SystemExit("Aborted.")
-
-        if transform.source_code is not None:
-            if target_relpath.suffix in (".ipynb", ".Rmd", ".qmd"):
-                script_to_notebook(transform, target_relpath, bump_revision=True)
-            else:
-                target_relpath.write_text(transform.source_code)
-        else:
-            raise SystemExit("No source code available for this transform.")
-
-        logger.important(f"{transform.type} is here: {target_relpath}")
-
-        if with_env:
-            ln.settings.track_run_inputs = False
-            if (
-                transform.latest_run is not None
-                and transform.latest_run.environment is not None
-            ):
-                filepath_env_cache = transform.latest_run.environment.cache()
-                target_env_filename = (
-                    target_relpath.parent / f"{target_relpath.stem}__requirements.txt"
-                )
-                shutil.move(filepath_env_cache, target_env_filename)
-                logger.important(f"environment is here: {target_env_filename}")
-            else:
-                logger.warning("latest transform run with environment doesn't exist")
-
-        return target_relpath
-    elif entity == "artifact":
-        ln.settings.track_run_inputs = False
-
-        if query_by_uid:
-            # we don't use .get here because DoesNotExist is hard to catch
-            # due to private django API
-            artifacts = ln.Artifact.filter(uid__startswith=uid)
-        else:
-            artifacts = ln.Artifact.filter(key=key)
-
-        if (n_artifacts := len(artifacts)) == 0:
-            err_msg = f"uid={uid}" if query_by_uid else f"key={key}"
-            raise SystemExit(f"Artifact with {err_msg} does not exist.")
-
-        if n_artifacts > 1:
-            artifacts = artifacts.order_by("-created_at")
-        artifact = artifacts.first()
-
-        cache_path = artifact.cache()
-        logger.important(f"artifact is here: {cache_path}")
-        return cache_path