lamin_cli 1.2.0__py2.py3-none-any.whl → 1.3.0__py2.py3-none-any.whl

lamin_cli/__init__.py

@@ -1,3 +1,3 @@
 """Lamin CLI."""
 
-__version__ = "1.2.0"
+__version__ = "1.3.0"

lamin_cli/__main__.py

@@ -9,6 +9,13 @@ from functools import wraps
 from importlib.metadata import PackageNotFoundError, version
 from typing import TYPE_CHECKING
 
+from lamindb_setup._init_instance import (
+    DOC_DB,
+    DOC_INSTANCE_NAME,
+    DOC_MODULES,
+    DOC_STORAGE_ARG,
+)
+
 if TYPE_CHECKING:
     from collections.abc import Mapping
 
@@ -41,12 +48,7 @@ else:
         "lamin": [
             {
                 "name": "Connect to an instance",
-                "commands": [
-                    "connect",
-                    "disconnect",
-                    "info",
-                    "init",
-                ],
+                "commands": ["connect", "disconnect", "info", "init", "run"],
             },
             {
                 "name": "Read & write data",
@@ -155,24 +157,20 @@ def schema_to_modules_callback(ctx, param, value):
 
 # fmt: off
 @main.command()
-@click.option("--storage", type=str, help="Local directory, s3://bucket_name, gs://bucket_name.")
-@click.option("--db", type=str, default=None, help="Postgres database connection URL, do not pass for SQLite.")
-@click.option("--modules", type=str, default=None, help="Comma-separated string of schema modules.")
-@click.option("--name", type=str, default=None, help="The instance name.")
-@click.option("--schema", type=str, default=None, help="[DEPRECATED] Use --modules instead.", callback=schema_to_modules_callback)
+@click.option("--storage", type=str, default = ".", help=DOC_STORAGE_ARG)
+@click.option("--name", type=str, default=None, help=DOC_INSTANCE_NAME)
+@click.option("--db", type=str, default=None, help=DOC_DB)
+@click.option("--modules", type=str, default=None, help=DOC_MODULES)
 # fmt: on
 def init(
     storage: str,
+    name: str | None,
     db: str | None,
     modules: str | None,
-    name: str | None,
-    schema: str | None,
 ):
     """Init an instance."""
     from lamindb_setup._init_instance import init as init_
 
-    modules = modules if modules is not None else schema
-
     return init_(storage=storage, db=db, modules=modules, name=name)
 
 
@@ -188,6 +186,8 @@ def connect(instance: str):
     `lamin connect` switches
     {attr}`~lamindb.setup.core.SetupSettings.auto_connect` to `True` so that you
     auto-connect in a Python session upon importing `lamindb`.
+
+    For manually connecting in a Python session, use {func}`~lamindb.connect`.
     """
     from lamindb_setup import connect as connect_
     from lamindb_setup import settings as settings_
@@ -307,30 +307,75 @@ def get(entity: str, uid: str | None = None, key: str | None = None):
 
 
 @main.command()
-@click.argument("filepath", type=click.Path(exists=True, dir_okay=True, file_okay=True))
+@click.argument("path", type=click.Path(exists=True, dir_okay=True, file_okay=True))
 @click.option("--key", type=str, default=None)
 @click.option("--description", type=str, default=None)
+@click.option("--stem-uid", type=str, default=None)
 @click.option("--registry", type=str, default=None)
-def save(filepath: str, key: str, description: str, registry: str):
+def save(path: str, key: str, description: str, stem_uid: str, registry: str):
     """Save a file or folder.
 
-    Defaults to saving `.py` and `.ipynb` as {class}`~lamindb.Transform` and
-    other file types and folders as {class}`~lamindb.Artifact`.
-
-    You can save a `.py` or `.ipynb` file as an {class}`~lamindb.Artifact` by
-    passing `--registry artifact`.
+    Defaults to saving `.py`, `.ipynb`, `.R`, `.Rmd`, and `.qmd` as {class}`~lamindb.Transform` and
+    other file types and folders as {class}`~lamindb.Artifact`. You can save a `.py` or `.ipynb` file as
+    an {class}`~lamindb.Artifact` by passing `--registry artifact`.
     """
     from lamin_cli._save import save_from_filepath_cli
 
-    if save_from_filepath_cli(filepath, key, description, registry) is not None:
+    if save_from_filepath_cli(path, key, description, stem_uid, registry) is not None:
         sys.exit(1)
 
 
+@main.command()
+@click.argument("filepath", type=str)
+@click.option("--project", type=str, default=None, help="A valid project name or uid. When running on Modal, creates an app with the same name.", required=True)
+@click.option("--image-url", type=str, default=None, help="A URL to the base docker image to use.")
+@click.option("--packages", type=str, default="lamindb", help="A comma-separated list of additional packages to install.")
+@click.option("--cpu", type=float, default=None, help="Configuration for the CPU.")
+@click.option("--gpu", type=str, default=None, help="The type of GPU to use (only compatible with cuda images).")
+def run(filepath: str, project: str, image_url: str, packages: str, cpu: int, gpu: str | None):
+    """Run a compute job in the cloud.
+
+    This is an EXPERIMENTAL feature that enables to run a script on Modal.
+
+    Example: Given a valid project name "my_project".
+
+    ```
+    lamin run my_script.py --project my_project
+    ```
+    """
+    import shutil
+    from pathlib import Path
+
+    from lamin_cli.compute.modal import Runner
+
+    default_mount_dir = Path('./modal_mount_dir')
+    if not default_mount_dir.is_dir():
+        default_mount_dir.mkdir(parents=True, exist_ok=True)
+
+    shutil.copy(filepath, default_mount_dir)
+
+    filepath_in_mount_dir = Path(default_mount_dir) / Path(filepath).name
+
+    package_list = []
+    if packages:
+        package_list = [package.strip() for package in packages.split(',')]
+
+    runner = Runner(
+        local_mount_dir=default_mount_dir,
+        app_name=project,
+        packages=package_list,
+        image_url=image_url,
+        cpu=cpu,
+        gpu=gpu
+    )
+
+    runner.run(filepath_in_mount_dir)
+
+
 main.add_command(settings)
 main.add_command(cache)
 main.add_command(migrate)
 
-
 # https://stackoverflow.com/questions/57810659/automatically-generate-all-help-documentation-for-click-commands
 # https://claude.ai/chat/73c28487-bec3-4073-8110-50d1a2dd6b84
 def _generate_help():

lamin_cli/_save.py

@@ -46,6 +46,7 @@ def save_from_filepath_cli(
     filepath: str | Path,
     key: str | None,
     description: str | None,
+    stem_uid: str | None,
     registry: str | None,
 ) -> str | None:
     import lamindb_setup as ln_setup
@@ -99,10 +100,21 @@ def save_from_filepath_cli(
 
     if registry == "artifact":
         ln.settings.creation.artifact_silence_missing_run_warning = True
-        if key is None and description is None:
+        revises = None
+        if stem_uid is not None:
+            revises = (
+                ln.Artifact.filter(uid__startswith=stem_uid)
+                .order_by("-created_at")
+                .first()
+            )
+            if revises is None:
+                raise ln.errors.InvalidArgument("The stem uid is not found.")
+        elif key is None and description is None:
             logger.error("Please pass a key or description via --key or --description")
             return "missing-key-or-description"
-        artifact = ln.Artifact(filepath, key=key, description=description).save()
+        artifact = ln.Artifact(
+            filepath, key=key, description=description, revises=revises
+        ).save()
         logger.important(f"saved: {artifact}")
         logger.important(f"storage path: {artifact.path}")
         if ln_setup.settings.storage.type == "s3":
@@ -125,6 +137,15 @@ def save_from_filepath_cli(
                 )
                 return "not-tracked-in-transform-registry"
         else:
+            revises = None
+            if stem_uid is not None:
+                revises = (
+                    ln.Transform.filter(uid__startswith=stem_uid)
+                    .order_by("-created_at")
+                    .first()
+                )
+                if revises is None:
+                    raise ln.errors.InvalidArgument("The stem uid is not found.")
             # TODO: build in the logic that queries for relative file paths
             # we have in Context; add tests for multiple versions
             transform = ln.Transform.filter(
@@ -135,6 +156,7 @@ def save_from_filepath_cli(
                     description=filepath.name,
                     key=filepath.name,
                     type="script" if filepath.suffix in {".R", ".py"} else "notebook",
+                    revises=revises,
                 ).save()
                 logger.important(f"created Transform('{transform.uid}')")
         # latest run of this transform by user
@@ -147,6 +169,11 @@ def save_from_filepath_cli(
             )
             if response != "y":
                 return "aborted-save-notebook-created-by-different-user"
+        if run is None and transform.key.endswith(".ipynb"):
+            run = ln.Run(transform=transform).save()
+            logger.important(
+                f"found no run, creating Run('{run.uid}') to display the html"
+            )
         return_code = save_context_core(
             run=run,
             transform=transform,

lamin_cli/compute/modal.py

@@ -0,0 +1,175 @@
+import os
+import subprocess
+import sys
+import threading
+from pathlib import Path
+
+import lamindb_setup as ln_setup
+import modal
+
+
+def run_script(path: Path) -> dict:
+    """Takes a path to a script for running it as a function through Modal."""
+    result = {"success": False, "output": "", "error": ""}
+
+    def stream_output(stream, capture_list):
+        """Read from stream line by line and print in real-time while also capturing to a list."""
+        for line in iter(stream.readline, ""):
+            print(line, end="")  # Print in real-time
+            capture_list.append(line)
+        stream.close()
+
+    if not path.exists():
+        raise FileNotFoundError(f"Script file not found: {path}")
+
+    try:
+        # Run the script using subprocess
+        process = subprocess.Popen(
+            [sys.executable, path.as_posix()],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,  # Line buffered
+        )
+
+        # Capture output and error while streaming stdout in real-time
+        stdout_lines: list[str] = []
+        stderr_lines: list[str] = []
+
+        # Create threads to handle stdout and stderr streams
+        stdout_thread = threading.Thread(
+            target=stream_output, args=(process.stdout, stdout_lines)
+        )
+        stderr_thread = threading.Thread(
+            target=stream_output, args=(process.stderr, stderr_lines)
+        )
+
+        # Set as daemon threads so they exit when the main program exits
+        stdout_thread.daemon = True
+        stderr_thread.daemon = True
+
+        # Start the threads
+        stdout_thread.start()
+        stderr_thread.start()
+
+        # Wait for the process to complete
+        return_code = process.wait()
+
+        # Wait for the threads to finish
+        stdout_thread.join()
+        stderr_thread.join()
+
+        # Join the captured output
+        stdout_output = "".join(stdout_lines)
+        stderr_output = "".join(stderr_lines)
+
+        # Check return code
+        if return_code == 0:
+            result["success"] = True
+            result["output"] = stdout_output
+        else:
+            result["error"] = stderr_output
+
+    except Exception as e:
+        import traceback
+
+        result["error"] = str(e) + "\n" + traceback.format_exc()
+    return result
+
+
+class Runner:
+    def __init__(
+        self,
+        app_name: str,
+        local_mount_dir: str | Path = "./scripts",
+        remote_mount_dir: str | Path = "/scripts",
+        image_url: str | None = None,
+        packages: list[str] | None = None,
+        cpu: float | None = None,
+        gpu: str | None = None,
+    ):
+        self.app_name = app_name  # we use the LaminDB project name as the app name
+        self.app = self.create_modal_app(app_name)
+
+        self.local_mount_dir = local_mount_dir
+        self.remote_mount_dir = remote_mount_dir
+
+        self.image = self.create_modal_image(
+            local_dir=local_mount_dir, packages=packages, image_url=image_url
+        )
+
+        local_secrets = self._configure_local_secrets()
+
+        self.modal_function = self.app.function(
+            image=self.image, cpu=cpu, gpu=gpu, secrets=[local_secrets]
+        )(run_script)
+
+    def run(self, script_local_path: Path) -> None:
+        script_remote_path = self.local_to_remote_path(str(script_local_path))
+        with modal.enable_output(show_progress=True):  # Prints out modal logs
+            with self.app.run():
+                self.modal_function.remote(Path(script_remote_path))
+
+    def create_modal_app(self, app_name: str) -> modal.App:
+        app = modal.App(app_name)
+        return app
+
+    def local_to_remote_path(self, local_path: str | Path) -> str:
+        local_path = Path(local_path).absolute()
+        local_mount_dir = Path(self.local_mount_dir).absolute()
+        remote_mount_dir = Path(self.remote_mount_dir)
+
+        # Check if local_path is inside local_mount_dir
+        try:
+            # This will raise ValueError if local_path is not relative to local_mount_dir
+            relative_path = local_path.relative_to(local_mount_dir)
+        except ValueError as err:
+            raise ValueError(
+                f"Local path '{local_path}' is not inside the mount directory '{local_mount_dir}'"
+            ) from err
+
+        # Join remote_mount_dir with the relative path
+        remote_path = remote_mount_dir / relative_path
+
+        # Return as string with normalized separators
+        return remote_path.as_posix()
+
+    def _configure_local_secrets(self) -> dict:
+        if ln_setup.settings.user.api_key is None:
+            raise ValueError("Please authenticate via: lamin login")
+
+        all_env_variables = {
+            "LAMIN_API_KEY": ln_setup.settings.user.api_key,
+            "LAMIN_CURRENT_PROJECT": self.app_name,
+            "LAMIN_CURRENT_INSTANCE": ln_setup.settings.instance.slug,
+        }
+        local_secrets = modal.Secret.from_dict(all_env_variables)
+        return local_secrets
+
+    def create_modal_image(
+        self,
+        python_version: str = "3.12",
+        packages: list[str] | None = None,
+        local_dir: str | Path = "./scripts",
+        remote_dir: str = "/scripts/",
+        image_url: str | None = None,
+        env_variables: dict | None = None,
+    ) -> modal.Image:
+        if env_variables is None:
+            env_variables = {}
+        if packages is None:
+            packages = ["lamindb"]
+        else:
+            packages.append("lamindb")  # Append lamindb to the list of packages
+
+        if image_url is None:
+            image = modal.Image.debian_slim(python_version=python_version)
+        else:
+            image = modal.Image.from_registry(image_url, add_python=python_version)
+        return (
+            image.pip_install(packages)
+            .env(env_variables)
+            .add_local_python_source("lamindb", "lamindb_setup", copy=True)
+            .run_commands("lamin settings set auto-connect true")
+            .add_local_dir(local_dir, remote_dir)
+        )

{lamin_cli-1.2.0.dist-info → lamin_cli-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamin_cli
-Version: 1.2.0
+Version: 1.3.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

lamin_cli-1.3.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+lamin_cli/__init__.py,sha256=M6lfPWq9Iw7_uUcLoHjPp1SVuvhsIU0NkQQDt-EYpIc,40
+lamin_cli/__main__.py,sha256=2CZdjlQK0nHrMyDpNvKn7iQ34Owlbwhl4mmavc5yXgc,12392
+lamin_cli/_cache.py,sha256=oplwE8AcS_9PYptQUZxff2qTIdNFS81clGPkJNWk098,800
+lamin_cli/_load.py,sha256=lMhV9AMkybjvj4VChJE_v7IMy6qGqisFlo40BJUibsA,8087
+lamin_cli/_migration.py,sha256=xQi6mwnpBzY5wcv1-TJhveD7a3XJIlpiYx6Z3AJ1NF0,1063
+lamin_cli/_save.py,sha256=JQQxby2PP6SfUb1FZhnwNnp0fkBqTj-UnbGql4oPt8g,6948
+lamin_cli/_settings.py,sha256=O2tecCf5EIZu98ima4DTJujo4KuywckOLgw8c-Ke3dY,1142
+lamin_cli/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lamin_cli/compute/modal.py,sha256=QnR7GyyvWWWkLnou95HxS9xxSQfw1k-SiefM_qRVnU0,6010
+lamin_cli-1.3.0.dist-info/entry_points.txt,sha256=Qms85i9cZPlu-U7RnVZhFsF7vJ9gaLZUFkCjcGcXTpg,49
+lamin_cli-1.3.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamin_cli-1.3.0.dist-info/WHEEL,sha256=ssQ84EZ5gH1pCOujd3iW7HClo_O_aDaClUbX4B8bjKY,100
+lamin_cli-1.3.0.dist-info/METADATA,sha256=m6_U_cQGfZ-2HjU-u_1frMM-Q6ATjQCObBkiebqNzMs,337
+lamin_cli-1.3.0.dist-info/RECORD,,

lamin_cli-1.2.0.dist-info/RECORD

@@ -1,12 +0,0 @@
-lamin_cli/__init__.py,sha256=2kLtbQt2_3KEeiaePHI1brXcTblfz2N7yIGKsKcAtb4,40
-lamin_cli/__main__.py,sha256=a14UZj5xS66nZXdL_dUkU9M5AAWvs5unQ9lHJx1S5fI,10839
-lamin_cli/_cache.py,sha256=oplwE8AcS_9PYptQUZxff2qTIdNFS81clGPkJNWk098,800
-lamin_cli/_load.py,sha256=lMhV9AMkybjvj4VChJE_v7IMy6qGqisFlo40BJUibsA,8087
-lamin_cli/_migration.py,sha256=xQi6mwnpBzY5wcv1-TJhveD7a3XJIlpiYx6Z3AJ1NF0,1063
-lamin_cli/_save.py,sha256=bt873beNgog5naWITjPb61cjy00aeEtIv9lwqQttRGI,5908
-lamin_cli/_settings.py,sha256=O2tecCf5EIZu98ima4DTJujo4KuywckOLgw8c-Ke3dY,1142
-lamin_cli-1.2.0.dist-info/entry_points.txt,sha256=Qms85i9cZPlu-U7RnVZhFsF7vJ9gaLZUFkCjcGcXTpg,49
-lamin_cli-1.2.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamin_cli-1.2.0.dist-info/WHEEL,sha256=ssQ84EZ5gH1pCOujd3iW7HClo_O_aDaClUbX4B8bjKY,100
-lamin_cli-1.2.0.dist-info/METADATA,sha256=22dRlSUzNnNcaapFD-faPqMrNG5Fi9KZLjuHVe06hEc,337
-lamin_cli-1.2.0.dist-info/RECORD,,