lamin_cli 1.2.0__tar.gz → 1.3.0__tar.gz

lamin_cli-1.3.0/.github/workflows/build.yml

@@ -0,0 +1,82 @@
+name: build
+
+on:
+  push:
+    branches: [release]
+  pull_request:
+
+jobs:
+  pre-filter:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: dorny/paths-filter@v3
+        id: changes
+        if: github.event_name != 'push'
+        with:
+          filters: |
+            modal:
+              - 'lamin_cli/compute/modal.py'
+              - 'tests/modal/**'
+
+      - id: set-matrix
+        shell: bash
+        run: |
+          BASE_GROUPS=$(jq -n -c '[]')
+
+          if [[ "${{ github.event_name }}" == "push" || "${{ steps.changes.outputs.modal }}" == "true" ]]; then
+            # Run everything on push or when modal paths change
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: ($groups + ["modal"])}')
+          else
+            # Otherwise only run base groups
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: $groups}')
+          fi
+
+          # Output as single line for GitHub Actions
+          echo "matrix=$(echo "$MATRIX" | jq -c .)" >> $GITHUB_OUTPUT
+
+          # Pretty print for debugging
+          echo "Generated matrix:"
+          echo "$MATRIX" | jq .
+
+  test:
+    needs: pre-filter
+    runs-on: ubuntu-latest
+    env:
+      LAMIN_API_KEY: ${{ secrets.LAMIN_API_KEY_TESTUSER1 }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.pre-filter.outputs.matrix)}}
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: cache pre-commit
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
+
+      - run: pip install git+https://github.com/laminlabs/laminci
+
+      - run: uv pip install --system modal pytest
+
+      - run: modal token set --token-id ${{ secrets.MODAL_DEV_TOKEN_ID }} --token-secret ${{ secrets.MODAL_DEV_TOKEN_SECRET }}
+
+      - run: nox -s setup
+
+      - run: lamin login
+
+      - run: pytest tests/modal

{lamin_cli-1.2.0 → lamin_cli-1.3.0}/.gitignore

@@ -1,4 +1,5 @@
 # LaminDB
+modal_mount_dir/
 lamindb_docs/
 _build
 mydata/

{lamin_cli-1.2.0 → lamin_cli-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamin_cli
-Version: 1.2.0
+Version: 1.3.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

lamin_cli-1.3.0/lamin_cli/__init__.py

@@ -0,0 +1,3 @@
+"""Lamin CLI."""
+
+__version__ = "1.3.0"

{lamin_cli-1.2.0 → lamin_cli-1.3.0}/lamin_cli/__main__.py

@@ -9,6 +9,13 @@ from functools import wraps
 from importlib.metadata import PackageNotFoundError, version
 from typing import TYPE_CHECKING
 
+from lamindb_setup._init_instance import (
+    DOC_DB,
+    DOC_INSTANCE_NAME,
+    DOC_MODULES,
+    DOC_STORAGE_ARG,
+)
+
 if TYPE_CHECKING:
     from collections.abc import Mapping
 
@@ -41,12 +48,7 @@ else:
         "lamin": [
             {
                 "name": "Connect to an instance",
-                "commands": [
-                    "connect",
-                    "disconnect",
-                    "info",
-                    "init",
-                ],
+                "commands": ["connect", "disconnect", "info", "init", "run"],
             },
             {
                 "name": "Read & write data",
@@ -155,24 +157,20 @@ def schema_to_modules_callback(ctx, param, value):
 
 # fmt: off
 @main.command()
-@click.option("--storage", type=str, help="Local directory, s3://bucket_name, gs://bucket_name.")
-@click.option("--db", type=str, default=None, help="Postgres database connection URL, do not pass for SQLite.")
-@click.option("--modules", type=str, default=None, help="Comma-separated string of schema modules.")
-@click.option("--name", type=str, default=None, help="The instance name.")
-@click.option("--schema", type=str, default=None, help="[DEPRECATED] Use --modules instead.", callback=schema_to_modules_callback)
+@click.option("--storage", type=str, default = ".", help=DOC_STORAGE_ARG)
+@click.option("--name", type=str, default=None, help=DOC_INSTANCE_NAME)
+@click.option("--db", type=str, default=None, help=DOC_DB)
+@click.option("--modules", type=str, default=None, help=DOC_MODULES)
 # fmt: on
 def init(
     storage: str,
+    name: str | None,
     db: str | None,
     modules: str | None,
-    name: str | None,
-    schema: str | None,
 ):
     """Init an instance."""
     from lamindb_setup._init_instance import init as init_
 
-    modules = modules if modules is not None else schema
-
     return init_(storage=storage, db=db, modules=modules, name=name)
 
 
@@ -188,6 +186,8 @@ def connect(instance: str):
     `lamin connect` switches
     {attr}`~lamindb.setup.core.SetupSettings.auto_connect` to `True` so that you
     auto-connect in a Python session upon importing `lamindb`.
+
+    For manually connecting in a Python session, use {func}`~lamindb.connect`.
     """
     from lamindb_setup import connect as connect_
     from lamindb_setup import settings as settings_
@@ -307,30 +307,75 @@ def get(entity: str, uid: str | None = None, key: str | None = None):
 
 
 @main.command()
-@click.argument("filepath", type=click.Path(exists=True, dir_okay=True, file_okay=True))
+@click.argument("path", type=click.Path(exists=True, dir_okay=True, file_okay=True))
 @click.option("--key", type=str, default=None)
 @click.option("--description", type=str, default=None)
+@click.option("--stem-uid", type=str, default=None)
 @click.option("--registry", type=str, default=None)
-def save(filepath: str, key: str, description: str, registry: str):
+def save(path: str, key: str, description: str, stem_uid: str, registry: str):
     """Save a file or folder.
 
-    Defaults to saving `.py` and `.ipynb` as {class}`~lamindb.Transform` and
-    other file types and folders as {class}`~lamindb.Artifact`.
-
-    You can save a `.py` or `.ipynb` file as an {class}`~lamindb.Artifact` by
-    passing `--registry artifact`.
+    Defaults to saving `.py`, `.ipynb`, `.R`, `.Rmd`, and `.qmd` as {class}`~lamindb.Transform` and
+    other file types and folders as {class}`~lamindb.Artifact`. You can save a `.py` or `.ipynb` file as
+    an {class}`~lamindb.Artifact` by passing `--registry artifact`.
     """
     from lamin_cli._save import save_from_filepath_cli
 
-    if save_from_filepath_cli(filepath, key, description, registry) is not None:
+    if save_from_filepath_cli(path, key, description, stem_uid, registry) is not None:
         sys.exit(1)
 
 
+@main.command()
+@click.argument("filepath", type=str)
+@click.option("--project", type=str, default=None, help="A valid project name or uid. When running on Modal, creates an app with the same name.", required=True)
+@click.option("--image-url", type=str, default=None, help="A URL to the base docker image to use.")
+@click.option("--packages", type=str, default="lamindb", help="A comma-separated list of additional packages to install.")
+@click.option("--cpu", type=float, default=None, help="Configuration for the CPU.")
+@click.option("--gpu", type=str, default=None, help="The type of GPU to use (only compatible with cuda images).")
+def run(filepath: str, project: str, image_url: str, packages: str, cpu: int, gpu: str | None):
+    """Run a compute job in the cloud.
+
+    This is an EXPERIMENTAL feature that enables to run a script on Modal.
+
+    Example: Given a valid project name "my_project".
+
+    ```
+    lamin run my_script.py --project my_project
+    ```
+    """
+    import shutil
+    from pathlib import Path
+
+    from lamin_cli.compute.modal import Runner
+
+    default_mount_dir = Path('./modal_mount_dir')
+    if not default_mount_dir.is_dir():
+        default_mount_dir.mkdir(parents=True, exist_ok=True)
+
+    shutil.copy(filepath, default_mount_dir)
+
+    filepath_in_mount_dir = Path(default_mount_dir) / Path(filepath).name
+
+    package_list = []
+    if packages:
+        package_list = [package.strip() for package in packages.split(',')]
+
+    runner = Runner(
+        local_mount_dir=default_mount_dir,
+        app_name=project,
+        packages=package_list,
+        image_url=image_url,
+        cpu=cpu,
+        gpu=gpu
+    )
+
+    runner.run(filepath_in_mount_dir)
+
+
 main.add_command(settings)
 main.add_command(cache)
 main.add_command(migrate)
 
-
 # https://stackoverflow.com/questions/57810659/automatically-generate-all-help-documentation-for-click-commands
 # https://claude.ai/chat/73c28487-bec3-4073-8110-50d1a2dd6b84
 def _generate_help():

{lamin_cli-1.2.0 → lamin_cli-1.3.0}/lamin_cli/_save.py

@@ -46,6 +46,7 @@ def save_from_filepath_cli(
     filepath: str | Path,
     key: str | None,
     description: str | None,
+    stem_uid: str | None,
     registry: str | None,
 ) -> str | None:
     import lamindb_setup as ln_setup
@@ -99,10 +100,21 @@ def save_from_filepath_cli(
 
     if registry == "artifact":
         ln.settings.creation.artifact_silence_missing_run_warning = True
-        if key is None and description is None:
+        revises = None
+        if stem_uid is not None:
+            revises = (
+                ln.Artifact.filter(uid__startswith=stem_uid)
+                .order_by("-created_at")
+                .first()
+            )
+            if revises is None:
+                raise ln.errors.InvalidArgument("The stem uid is not found.")
+        elif key is None and description is None:
             logger.error("Please pass a key or description via --key or --description")
             return "missing-key-or-description"
-        artifact = ln.Artifact(filepath, key=key, description=description).save()
+        artifact = ln.Artifact(
+            filepath, key=key, description=description, revises=revises
+        ).save()
         logger.important(f"saved: {artifact}")
         logger.important(f"storage path: {artifact.path}")
         if ln_setup.settings.storage.type == "s3":
@@ -125,6 +137,15 @@ def save_from_filepath_cli(
                 )
                 return "not-tracked-in-transform-registry"
         else:
+            revises = None
+            if stem_uid is not None:
+                revises = (
+                    ln.Transform.filter(uid__startswith=stem_uid)
+                    .order_by("-created_at")
+                    .first()
+                )
+                if revises is None:
+                    raise ln.errors.InvalidArgument("The stem uid is not found.")
             # TODO: build in the logic that queries for relative file paths
             # we have in Context; add tests for multiple versions
             transform = ln.Transform.filter(
@@ -135,6 +156,7 @@ def save_from_filepath_cli(
                     description=filepath.name,
                     key=filepath.name,
                     type="script" if filepath.suffix in {".R", ".py"} else "notebook",
+                    revises=revises,
                 ).save()
                 logger.important(f"created Transform('{transform.uid}')")
         # latest run of this transform by user
@@ -147,6 +169,11 @@ def save_from_filepath_cli(
             )
             if response != "y":
                 return "aborted-save-notebook-created-by-different-user"
+        if run is None and transform.key.endswith(".ipynb"):
+            run = ln.Run(transform=transform).save()
+            logger.important(
+                f"found no run, creating Run('{run.uid}') to display the html"
+            )
         return_code = save_context_core(
             run=run,
             transform=transform,

lamin_cli-1.3.0/lamin_cli/compute/modal.py

@@ -0,0 +1,175 @@
+import os
+import subprocess
+import sys
+import threading
+from pathlib import Path
+
+import lamindb_setup as ln_setup
+import modal
+
+
+def run_script(path: Path) -> dict:
+    """Takes a path to a script for running it as a function through Modal."""
+    result = {"success": False, "output": "", "error": ""}
+
+    def stream_output(stream, capture_list):
+        """Read from stream line by line and print in real-time while also capturing to a list."""
+        for line in iter(stream.readline, ""):
+            print(line, end="")  # Print in real-time
+            capture_list.append(line)
+        stream.close()
+
+    if not path.exists():
+        raise FileNotFoundError(f"Script file not found: {path}")
+
+    try:
+        # Run the script using subprocess
+        process = subprocess.Popen(
+            [sys.executable, path.as_posix()],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,  # Line buffered
+        )
+
+        # Capture output and error while streaming stdout in real-time
+        stdout_lines: list[str] = []
+        stderr_lines: list[str] = []
+
+        # Create threads to handle stdout and stderr streams
+        stdout_thread = threading.Thread(
+            target=stream_output, args=(process.stdout, stdout_lines)
+        )
+        stderr_thread = threading.Thread(
+            target=stream_output, args=(process.stderr, stderr_lines)
+        )
+
+        # Set as daemon threads so they exit when the main program exits
+        stdout_thread.daemon = True
+        stderr_thread.daemon = True
+
+        # Start the threads
+        stdout_thread.start()
+        stderr_thread.start()
+
+        # Wait for the process to complete
+        return_code = process.wait()
+
+        # Wait for the threads to finish
+        stdout_thread.join()
+        stderr_thread.join()
+
+        # Join the captured output
+        stdout_output = "".join(stdout_lines)
+        stderr_output = "".join(stderr_lines)
+
+        # Check return code
+        if return_code == 0:
+            result["success"] = True
+            result["output"] = stdout_output
+        else:
+            result["error"] = stderr_output
+
+    except Exception as e:
+        import traceback
+
+        result["error"] = str(e) + "\n" + traceback.format_exc()
+    return result
+
+
+class Runner:
+    def __init__(
+        self,
+        app_name: str,
+        local_mount_dir: str | Path = "./scripts",
+        remote_mount_dir: str | Path = "/scripts",
+        image_url: str | None = None,
+        packages: list[str] | None = None,
+        cpu: float | None = None,
+        gpu: str | None = None,
+    ):
+        self.app_name = app_name  # we use the LaminDB project name as the app name
+        self.app = self.create_modal_app(app_name)
+
+        self.local_mount_dir = local_mount_dir
+        self.remote_mount_dir = remote_mount_dir
+
+        self.image = self.create_modal_image(
+            local_dir=local_mount_dir, packages=packages, image_url=image_url
+        )
+
+        local_secrets = self._configure_local_secrets()
+
+        self.modal_function = self.app.function(
+            image=self.image, cpu=cpu, gpu=gpu, secrets=[local_secrets]
+        )(run_script)
+
+    def run(self, script_local_path: Path) -> None:
+        script_remote_path = self.local_to_remote_path(str(script_local_path))
+        with modal.enable_output(show_progress=True):  # Prints out modal logs
+            with self.app.run():
+                self.modal_function.remote(Path(script_remote_path))
+
+    def create_modal_app(self, app_name: str) -> modal.App:
+        app = modal.App(app_name)
+        return app
+
+    def local_to_remote_path(self, local_path: str | Path) -> str:
+        local_path = Path(local_path).absolute()
+        local_mount_dir = Path(self.local_mount_dir).absolute()
+        remote_mount_dir = Path(self.remote_mount_dir)
+
+        # Check if local_path is inside local_mount_dir
+        try:
+            # This will raise ValueError if local_path is not relative to local_mount_dir
+            relative_path = local_path.relative_to(local_mount_dir)
+        except ValueError as err:
+            raise ValueError(
+                f"Local path '{local_path}' is not inside the mount directory '{local_mount_dir}'"
+            ) from err
+
+        # Join remote_mount_dir with the relative path
+        remote_path = remote_mount_dir / relative_path
+
+        # Return as string with normalized separators
+        return remote_path.as_posix()
+
+    def _configure_local_secrets(self) -> dict:
+        if ln_setup.settings.user.api_key is None:
+            raise ValueError("Please authenticate via: lamin login")
+
+        all_env_variables = {
+            "LAMIN_API_KEY": ln_setup.settings.user.api_key,
+            "LAMIN_CURRENT_PROJECT": self.app_name,
+            "LAMIN_CURRENT_INSTANCE": ln_setup.settings.instance.slug,
+        }
+        local_secrets = modal.Secret.from_dict(all_env_variables)
+        return local_secrets
+
+    def create_modal_image(
+        self,
+        python_version: str = "3.12",
+        packages: list[str] | None = None,
+        local_dir: str | Path = "./scripts",
+        remote_dir: str = "/scripts/",
+        image_url: str | None = None,
+        env_variables: dict | None = None,
+    ) -> modal.Image:
+        if env_variables is None:
+            env_variables = {}
+        if packages is None:
+            packages = ["lamindb"]
+        else:
+            packages.append("lamindb")  # Append lamindb to the list of packages
+
+        if image_url is None:
+            image = modal.Image.debian_slim(python_version=python_version)
+        else:
+            image = modal.Image.from_registry(image_url, add_python=python_version)
+        return (
+            image.pip_install(packages)
+            .env(env_variables)
+            .add_local_python_source("lamindb", "lamindb_setup", copy=True)
+            .run_commands("lamin settings set auto-connect true")
+            .add_local_dir(local_dir, remote_dir)
+        )

lamin_cli-1.3.0/noxfile.py

@@ -0,0 +1,13 @@
+import os
+
+import nox
+from laminci.nox import install_lamindb
+
+IS_PR = os.getenv("GITHUB_EVENT_NAME") != "push"
+nox.options.default_venv_backend = "none"
+
+
+@nox.session
+def setup(session):
+    branch = "main" if IS_PR else "release"
+    install_lamindb(session, branch=branch)

{lamin_cli-1.2.0/tests → lamin_cli-1.3.0/tests/core}/test_multi_process.py

@@ -2,7 +2,7 @@ import subprocess
 from multiprocessing import Process
 from pathlib import Path
 
-scripts_dir = Path(__file__).parent.resolve() / "scripts"
+scripts_dir = Path(__file__).parent.parent.resolve() / "scripts"
 
 
 def run_script():

{lamin_cli-1.2.0/tests → lamin_cli-1.3.0/tests/core}/test_save_files.py

@@ -3,7 +3,7 @@ from pathlib import Path
 
 import lamindb_setup as ln_setup
 
-test_file = Path(__file__).parent.parent.resolve() / ".gitignore"
+test_file = Path(__file__).parent.parent.parent.resolve() / ".gitignore"
 
 
 def test_save_file():

{lamin_cli-1.2.0/tests → lamin_cli-1.3.0/tests/core}/test_save_notebooks.py

@@ -184,9 +184,7 @@ print("my consecutive cell")
     new_path = notebook_path.with_name("new_name.ipynb")
     os.system(f"cp {notebook_path} {new_path}")
 
-    # upon re-running it, the user is asked to create a new stem uid
-    with pytest.raises(CellExecutionError) as error:
-        nbproject_test.execute_notebooks(new_path, print_outputs=True)
-
-    print(error.exconly())
-    assert "clashes with the existing key" in error.exconly()
+    # upon re-running it, the notebook name is updated
+    nbproject_test.execute_notebooks(new_path, print_outputs=True)
+    transform = ln.Transform.get("hlsFXswrJjtt0001")
+    assert "new_name.ipynb" in transform.key

{lamin_cli-1.2.0/tests → lamin_cli-1.3.0/tests/core}/test_save_r_code.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 import lamindb as ln
 
-scripts_dir = Path(__file__).parent.resolve() / "scripts"
+scripts_dir = Path(__file__).parent.parent.resolve() / "scripts"
 
 
 def test_run_save_cache():

{lamin_cli-1.2.0/tests → lamin_cli-1.3.0/tests/core}/test_save_scripts.py

@@ -5,7 +5,7 @@ from pathlib import Path
 import lamindb as ln
 from lamindb_setup import settings
 
-scripts_dir = Path(__file__).parent.resolve() / "scripts"
+scripts_dir = Path(__file__).parent.parent.resolve() / "scripts"
 
 
 def test_save_without_uid():

lamin_cli-1.3.0/tests/modal/test_modal.py

@@ -0,0 +1,19 @@
+import subprocess
+from pathlib import Path
+
+scripts_dir = Path(__file__).parent.parent.resolve() / "scripts"
+
+
+def test_run_on_modal():
+    filepath = scripts_dir / "run-track-and-finish.py"
+
+    subprocess.run("lamin connect laminlabs/lamindata", shell=True, check=True)
+    result = subprocess.run(
+        f"lamin run {filepath} --project 1QLbS6N7wwiL",
+        shell=True,
+        capture_output=True,
+    )
+    print(result.stdout.decode())
+    assert result.returncode == 0
+    assert "hello!" in result.stdout.decode()
+    assert "finished Run" in result.stdout.decode()

{lamin_cli-1.2.0 → lamin_cli-1.3.0}/tests/scripts/run-track-and-finish.py

@@ -1,9 +1,5 @@
 import lamindb as ln
 
-# purposefully do not pass uid
-ln.track()
-
-
 if __name__ == "__main__":
     # we're using new_run here to mock the notebook situation
     # and cover the look up of an existing run in the tests

lamin_cli-1.2.0/lamin_cli/__init__.py

@@ -1,3 +0,0 @@
-"""Lamin CLI."""
-
-__version__ = "1.2.0"