lamin_cli 1.2.0__tar.gz → 1.4.0__tar.gz

lamin_cli-1.4.0/.github/workflows/build.yml

@@ -0,0 +1,82 @@
+name: build
+
+on:
+  push:
+    branches: [release]
+  pull_request:
+
+jobs:
+  pre-filter:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: dorny/paths-filter@v3
+        id: changes
+        if: github.event_name != 'push'
+        with:
+          filters: |
+            modal:
+              - 'lamin_cli/compute/modal.py'
+              - 'tests/modal/**'
+
+      - id: set-matrix
+        shell: bash
+        run: |
+          BASE_GROUPS=$(jq -n -c '[]')
+
+          if [[ "${{ github.event_name }}" == "push" || "${{ steps.changes.outputs.modal }}" == "true" ]]; then
+            # Run everything on push or when modal paths change
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: ($groups + ["modal"])}')
+          else
+            # Otherwise only run base groups
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: $groups}')
+          fi
+
+          # Output as single line for GitHub Actions
+          echo "matrix=$(echo "$MATRIX" | jq -c .)" >> $GITHUB_OUTPUT
+
+          # Pretty print for debugging
+          echo "Generated matrix:"
+          echo "$MATRIX" | jq .
+
+  test:
+    needs: pre-filter
+    runs-on: ubuntu-latest
+    env:
+      LAMIN_API_KEY: ${{ secrets.LAMIN_API_KEY_TESTUSER1 }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.pre-filter.outputs.matrix)}}
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: cache pre-commit
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
+
+      - run: pip install git+https://github.com/laminlabs/laminci
+
+      - run: uv pip install --system modal pytest
+
+      - run: modal token set --token-id ${{ secrets.MODAL_DEV_TOKEN_ID }} --token-secret ${{ secrets.MODAL_DEV_TOKEN_SECRET }}
+
+      - run: nox -s setup
+
+      - run: lamin login
+
+      - run: pytest tests/modal

{lamin_cli-1.2.0 → lamin_cli-1.4.0}/.gitignore

@@ -1,4 +1,5 @@
 # LaminDB
+modal_mount_dir/
 lamindb_docs/
 _build
 mydata/

{lamin_cli-1.2.0 → lamin_cli-1.4.0}/.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
               tests
           )
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.14.1
+    rev: v1.15.0
     hooks:
       - id: mypy
         exclude: |

{lamin_cli-1.2.0 → lamin_cli-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamin_cli
-Version: 1.2.0
+Version: 1.4.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

lamin_cli-1.4.0/lamin_cli/__init__.py

@@ -0,0 +1,3 @@
+"""Lamin CLI."""
+
+__version__ = "1.4.0"

{lamin_cli-1.2.0 → lamin_cli-1.4.0}/lamin_cli/__main__.py

@@ -2,13 +2,22 @@ from __future__ import annotations
 
 import inspect
 import os
+import shutil
 import sys
 import warnings
 from collections import OrderedDict
 from functools import wraps
 from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
 from typing import TYPE_CHECKING
 
+from lamindb_setup._init_instance import (
+    DOC_DB,
+    DOC_INSTANCE_NAME,
+    DOC_MODULES,
+    DOC_STORAGE_ARG,
+)
+
 if TYPE_CHECKING:
     from collections.abc import Mapping
 
@@ -41,12 +50,7 @@ else:
         "lamin": [
             {
                 "name": "Connect to an instance",
-                "commands": [
-                    "connect",
-                    "disconnect",
-                    "info",
-                    "init",
-                ],
+                "commands": ["connect", "disconnect", "info", "init", "run"],
             },
             {
                 "name": "Read & write data",
@@ -155,24 +159,20 @@ def schema_to_modules_callback(ctx, param, value):
 
 # fmt: off
 @main.command()
-@click.option("--storage", type=str, help="Local directory, s3://bucket_name, gs://bucket_name.")
-@click.option("--db", type=str, default=None, help="Postgres database connection URL, do not pass for SQLite.")
-@click.option("--modules", type=str, default=None, help="Comma-separated string of schema modules.")
-@click.option("--name", type=str, default=None, help="The instance name.")
-@click.option("--schema", type=str, default=None, help="[DEPRECATED] Use --modules instead.", callback=schema_to_modules_callback)
+@click.option("--storage", type=str, default = ".", help=DOC_STORAGE_ARG)
+@click.option("--name", type=str, default=None, help=DOC_INSTANCE_NAME)
+@click.option("--db", type=str, default=None, help=DOC_DB)
+@click.option("--modules", type=str, default=None, help=DOC_MODULES)
 # fmt: on
 def init(
     storage: str,
+    name: str | None,
     db: str | None,
     modules: str | None,
-    name: str | None,
-    schema: str | None,
 ):
     """Init an instance."""
     from lamindb_setup._init_instance import init as init_
 
-    modules = modules if modules is not None else schema
-
     return init_(storage=storage, db=db, modules=modules, name=name)
 
 
@@ -188,6 +188,8 @@ def connect(instance: str):
     `lamin connect` switches
     {attr}`~lamindb.setup.core.SetupSettings.auto_connect` to `True` so that you
     auto-connect in a Python session upon importing `lamindb`.
+
+    For manually connecting in a Python session, use {func}`~lamindb.connect`.
     """
     from lamindb_setup import connect as connect_
     from lamindb_setup import settings as settings_
@@ -307,30 +309,79 @@ def get(entity: str, uid: str | None = None, key: str | None = None):
 
 
 @main.command()
-@click.argument("filepath", type=click.Path(exists=True, dir_okay=True, file_okay=True))
-@click.option("--key", type=str, default=None)
-@click.option("--description", type=str, default=None)
-@click.option("--registry", type=str, default=None)
-def save(filepath: str, key: str, description: str, registry: str):
+@click.argument("path", type=str)
+@click.option("--key", type=str, default=None, help="The key of the artifact or transform.")
+@click.option("--description", type=str, default=None, help="A description of the artifact or transform.")
+@click.option("--stem-uid", type=str, default=None, help="The stem uid of the artifact or transform.")
+@click.option("--project", type=str, default=None, help="A valid project name or uid.")
+@click.option("--registry", type=str, default=None, help="Either 'artifact' or 'transform'. If not passed, chooses based on path suffix.")
+def save(path: str, key: str, description: str, stem_uid: str, project: str, registry: str):
     """Save a file or folder.
 
-    Defaults to saving `.py` and `.ipynb` as {class}`~lamindb.Transform` and
-    other file types and folders as {class}`~lamindb.Artifact`.
+    Example: Given a valid project name "my_project".
+
+    ```
+    lamin save my_table.csv --key my_tables/my_table.csv --project my_project
+    ```
 
-    You can save a `.py` or `.ipynb` file as an {class}`~lamindb.Artifact` by
-    passing `--registry artifact`.
+    Note: Defaults to saving `.py`, `.ipynb`, `.R`, `.Rmd`, and `.qmd` as {class}`~lamindb.Transform` and
+    other file types and folders as {class}`~lamindb.Artifact`. You can enforce saving a file as
+    an {class}`~lamindb.Artifact` by passing `--registry artifact`.
     """
-    from lamin_cli._save import save_from_filepath_cli
+    from lamin_cli._save import save_from_path_cli
 
-    if save_from_filepath_cli(filepath, key, description, registry) is not None:
+    if save_from_path_cli(path, key, description, stem_uid, project, registry) is not None:
         sys.exit(1)
 
 
+@main.command()
+@click.argument("filepath", type=str)
+@click.option("--project", type=str, default=None, help="A valid project name or uid. When running on Modal, creates an app with the same name.", required=True)
+@click.option("--image-url", type=str, default=None, help="A URL to the base docker image to use.")
+@click.option("--packages", type=str, default="lamindb", help="A comma-separated list of additional packages to install.")
+@click.option("--cpu", type=float, default=None, help="Configuration for the CPU.")
+@click.option("--gpu", type=str, default=None, help="The type of GPU to use (only compatible with cuda images).")
+def run(filepath: str, project: str, image_url: str, packages: str, cpu: int, gpu: str | None):
+    """Run a compute job in the cloud.
+
+    This is an EXPERIMENTAL feature that enables to run a script on Modal.
+
+    Example: Given a valid project name "my_project".
+
+    ```
+    lamin run my_script.py --project my_project
+    ```
+    """
+    from lamin_cli.compute.modal import Runner
+
+    default_mount_dir = Path('./modal_mount_dir')
+    if not default_mount_dir.is_dir():
+        default_mount_dir.mkdir(parents=True, exist_ok=True)
+
+    shutil.copy(filepath, default_mount_dir)
+
+    filepath_in_mount_dir = default_mount_dir / Path(filepath).name
+
+    package_list = []
+    if packages:
+        package_list = [package.strip() for package in packages.split(',')]
+
+    runner = Runner(
+        local_mount_dir=default_mount_dir,
+        app_name=project,
+        packages=package_list,
+        image_url=image_url,
+        cpu=cpu,
+        gpu=gpu
+    )
+
+    runner.run(filepath_in_mount_dir)
+
+
 main.add_command(settings)
 main.add_command(cache)
 main.add_command(migrate)
 
-
 # https://stackoverflow.com/questions/57810659/automatically-generate-all-help-documentation-for-click-commands
 # https://claude.ai/chat/73c28487-bec3-4073-8110-50d1a2dd6b84
 def _generate_help():

{lamin_cli-1.2.0 → lamin_cli-1.4.0}/lamin_cli/_load.py

@@ -78,7 +78,9 @@ def load(
                     )
         if bump_revision:
             uid = transform.uid
-            if uid in new_content:
+            if (
+                uid in new_content
+            ):  # this only hits if it has the full uid, not for the stem uid
                 new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
                 new_content = new_content.replace(uid, new_uid)
                 logger.important(f"updated uid: {uid} → {new_uid}")

lamin_cli-1.4.0/lamin_cli/_save.py

@@ -0,0 +1,239 @@
+from __future__ import annotations
+
+import re
+import sys
+from typing import TYPE_CHECKING
+
+import click
+from lamin_utils import logger
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+def parse_uid_from_code(content: str, suffix: str) -> str | None:
+    if suffix == ".py":
+        track_pattern = re.compile(
+            r'ln\.track\(\s*(?:transform\s*=\s*)?(["\'])([a-zA-Z0-9]{12,16})\1'
+        )
+        uid_pattern = re.compile(r'\.context\.uid\s*=\s*["\']([^"\']+)["\']')
+    elif suffix == ".ipynb":
+        track_pattern = re.compile(
+            r'ln\.track\(\s*(?:transform\s*=\s*)?(?:\\"|\')([a-zA-Z0-9]{12,16})(?:\\"|\')'
+        )
+        # backward compat
+        uid_pattern = re.compile(r'\.context\.uid\s*=\s*\\["\']([^"\']+)\\["\']')
+    elif suffix in {".R", ".qmd", ".Rmd"}:
+        track_pattern = re.compile(
+            r'track\(\s*(?:transform\s*=\s*)?([\'"])([a-zA-Z0-9]{12,16})\1'
+        )
+        uid_pattern = None
+    else:
+        raise SystemExit(
+            "Only .py, .ipynb, .R, .qmd, .Rmd files are supported for saving"
+            " transforms."
+        )
+
+    # Search for matches in the entire file content
+    uid_match = track_pattern.search(content)
+    group_index = 1 if suffix == ".ipynb" else 2
+    uid = uid_match.group(group_index) if uid_match else None
+
+    if uid_pattern is not None and uid is None:
+        uid_match = uid_pattern.search(content)
+        uid = uid_match.group(1) if uid_match else None
+
+    return uid
+
+
+def save_from_path_cli(
+    path: Path | str,
+    key: str | None,
+    description: str | None,
+    stem_uid: str | None,
+    project: str | None,
+    registry: str | None,
+) -> str | None:
+    import lamindb_setup as ln_setup
+    from lamindb_setup.core.upath import LocalPathClasses, UPath, create_path
+
+    # this will be gone once we get rid of lamin load or enable loading multiple
+    # instances sequentially
+    auto_connect_state = ln_setup.settings.auto_connect
+    ln_setup.settings.auto_connect = True
+
+    import lamindb as ln
+
+    if not ln.setup.core.django.IS_SETUP:
+        sys.exit(-1)
+    from lamindb._finish import save_context_core
+
+    ln_setup.settings.auto_connect = auto_connect_state
+
+    # this allows to have the correct treatment of credentials in case of cloud paths
+    path = create_path(path)
+    # isinstance is needed to cast the type of path to UPath
+    # to avoid mypy erors
+    assert isinstance(path, UPath)
+    if not path.exists():
+        raise click.BadParameter(f"Path {path} does not exist", param_hint="path")
+
+    if registry is None:
+        suffixes_transform = {
+            "py": {".py", ".ipynb"},
+            "R": {".R", ".qmd", ".Rmd"},
+        }
+        registry = (
+            "transform"
+            if path.suffix in suffixes_transform["py"].union(suffixes_transform["R"])
+            else "artifact"
+        )
+
+    if project is not None:
+        project_record = ln.Project.filter(
+            ln.Q(name=project) | ln.Q(uid=project)
+        ).one_or_none()
+        if project_record is None:
+            raise ln.errors.InvalidArgument(
+                f"Project '{project}' not found, either create it with `ln.Project(name='...').save()` or fix typos."
+            )
+
+    is_cloud_path = not isinstance(path, LocalPathClasses)
+
+    if registry == "artifact":
+        ln.settings.creation.artifact_silence_missing_run_warning = True
+        revises = None
+        if stem_uid is not None:
+            revises = (
+                ln.Artifact.filter(uid__startswith=stem_uid)
+                .order_by("-created_at")
+                .first()
+            )
+            if revises is None:
+                raise ln.errors.InvalidArgument("The stem uid is not found.")
+
+        if is_cloud_path:
+            if key is not None:
+                logger.error("Do not pass --key for cloud paths")
+                return "key-with-cloud-path"
+        elif key is None and description is None:
+            logger.error("Please pass a key or description via --key or --description")
+            return "missing-key-or-description"
+
+        artifact = ln.Artifact(
+            path, key=key, description=description, revises=revises
+        ).save()
+        logger.important(f"saved: {artifact}")
+        logger.important(f"storage path: {artifact.path}")
+        if ln_setup.settings.storage.type == "s3":
+            logger.important(f"storage url: {artifact.path.to_url()}")
+        if project is not None:
+            artifact.projects.add(project_record)
+            logger.important(f"labeled with project: {project_record.name}")
+        if ln_setup.settings.instance.is_remote:
+            slug = ln_setup.settings.instance.slug
+            logger.important(f"go to: https://lamin.ai/{slug}/artifact/{artifact.uid}")
+        return None
+
+    if registry == "transform":
+        if is_cloud_path:
+            logger.error("Can not register a transform from a cloud path")
+            return "transform-with-cloud-path"
+
+        if path.suffix in {".qmd", ".Rmd"}:
+            html_file_exists = path.with_suffix(".html").exists()
+            nb_html_file_exists = path.with_suffix(".nb.html").exists()
+
+            if not html_file_exists and not nb_html_file_exists:
+                logger.error(
+                    f"Please export your {path.suffix} file as an html file here"
+                    f" {path.with_suffix('.html')}"
+                )
+                return "export-qmd-Rmd-as-html"
+            elif html_file_exists and nb_html_file_exists:
+                logger.error(
+                    f"Please delete one of\n - {path.with_suffix('.html')}\n -"
+                    f" {path.with_suffix('.nb.html')}"
+                )
+                return "delete-html-or-nb-html"
+
+        with path.open() as file:
+            content = file.read()
+        uid = parse_uid_from_code(content, path.suffix)
+
+        if uid is not None:
+            logger.important(f"mapped '{path.name}' on uid '{uid}'")
+            if len(uid) == 16:
+                # is full uid
+                transform = ln.Transform.filter(uid=uid).one_or_none()
+            else:
+                # is stem uid
+                if stem_uid is not None:
+                    assert stem_uid == uid, (
+                        "passed stem uid and parsed stem uid do not match"
+                    )
+                else:
+                    stem_uid = uid
+                transform = (
+                    ln.Transform.filter(uid__startswith=uid)
+                    .order_by("-created_at")
+                    .first()
+                )
+                if transform is None:
+                    uid = f"{stem_uid}0000"
+        else:
+            # TODO: account for folders and hash equivalence as we do in ln.track()
+            transform = ln.Transform.filter(key=path.name, is_latest=True).one_or_none()
+        revises = None
+        if stem_uid is not None:
+            revises = (
+                ln.Transform.filter(uid__startswith=stem_uid)
+                .order_by("-created_at")
+                .first()
+            )
+            if revises is None:
+                raise ln.errors.InvalidArgument("The stem uid is not found.")
+        if transform is None:
+            if path.suffix == ".ipynb":
+                from nbproject.dev import read_notebook
+                from nbproject.dev._meta_live import get_title
+
+                nb = read_notebook(path)
+                description = get_title(nb)
+            else:
+                description = None
+            transform = ln.Transform(
+                uid=uid,
+                description=description,
+                key=path.name,
+                type="script" if path.suffix in {".R", ".py"} else "notebook",
+                revises=revises,
+            ).save()
+            logger.important(f"created Transform('{transform.uid}')")
+        if project is not None:
+            transform.projects.add(project_record)
+            logger.important(f"labeled with project: {project_record.name}")
+        # latest run of this transform by user
+        run = ln.Run.filter(transform=transform).order_by("-started_at").first()
+        if run is not None and run.created_by.id != ln_setup.settings.user.id:
+            response = input(
+                "You are trying to save a transform created by another user: Source"
+                " and report files will be tagged with *your* user id. Proceed?"
+                " (y/n)"
+            )
+            if response != "y":
+                return "aborted-save-notebook-created-by-different-user"
+        if run is None and transform.key.endswith(".ipynb"):
+            run = ln.Run(transform=transform).save()
+            logger.important(
+                f"found no run, creating Run('{run.uid}') to display the html"
+            )
+        return_code = save_context_core(
+            run=run,
+            transform=transform,
+            filepath=path,
+            from_cli=True,
+        )
+        return return_code
+    else:
+        raise SystemExit("Allowed values for '--registry' are: 'artifact', 'transform'")