lamin_cli 1.9.0__tar.gz → 1.11.0__tar.gz

{lamin_cli-1.9.0 → lamin_cli-1.11.0}/.github/workflows/build.yml

@@ -11,7 +11,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
@@ -54,12 +54,12 @@ jobs:
       matrix: ${{fromJson(needs.pre-filter.outputs.matrix)}}
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           submodules: recursive
           fetch-depth: 0
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: 3.12
 

{lamin_cli-1.9.0 → lamin_cli-1.11.0}/.gitignore

@@ -22,7 +22,6 @@ docs/conf.py
 lamindb/setup/.env
 _secrets.py
 _configuration.py
-lamin.db
 docs/generated/*
 _docs_tmp*
 docs/guide/Laminopathic_nuclei.jpg
@@ -41,14 +40,6 @@ docs/biology/test-flow/
 docs/biology/test-scrna/
 docs/biology/test-registries/
 docs/biology/test-multimodal/
-test-inherit1
-test-inherit2
-test-search0
-test-search1
-test-search2
-test-search3
-test-search4
-test-search5
 test.ipynb
 
 # General
@@ -181,6 +172,9 @@ venv.bak/
 .dmypy.json
 dmypy.json
 
+# ruff
+.ruff_cache
+
 # Pyre type checker
 .pyre/
 

{lamin_cli-1.9.0 → lamin_cli-1.11.0}/.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -11,13 +11,13 @@ repos:
       - id: check-yaml
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.2
+    rev: v0.14.6
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix, --unsafe-fixes]
       - id: ruff-format
   - repo: https://github.com/rbubley/mirrors-prettier
-    rev: v3.5.1
+    rev: v3.6.2
     hooks:
       - id: prettier
   - repo: https://github.com/kynan/nbstripout
@@ -31,7 +31,7 @@ repos:
               tests
           )
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.15.0
+    rev: v1.18.2
     hooks:
       - id: mypy
         exclude: |

{lamin_cli-1.9.0 → lamin_cli-1.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamin_cli
-Version: 1.9.0
+Version: 1.11.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

{lamin_cli-1.9.0 → lamin_cli-1.11.0}/lamin_cli/__init__.py

@@ -2,10 +2,11 @@
 
 This is the command line interface for interacting with LaminDB & LaminHub.
 
-The interface is defined in `__main__.py`. The root API here is used by LaminR to replicate the CLI functionality.
+The interface is defined in `__main__.py`.
+The root API here is used by LaminR to replicate the CLI functionality.
 """
 
-__version__ = "1.9.0"
+__version__ = "1.11.0"
 
 from lamindb_setup import disconnect, logout
 from lamindb_setup._connect_instance import _connect_cli as connect
@@ -15,4 +16,12 @@ from lamindb_setup._setup_user import login
 from ._delete import delete
 from ._save import save
 
-__all__ = ["save", "init", "connect", "delete", "login", "disconnect"]
+__all__ = [
+    "save",
+    "init",
+    "connect",
+    "delete",
+    "login",
+    "logout",
+    "disconnect",
+]

{lamin_cli-1.9.0 → lamin_cli-1.11.0}/lamin_cli/__main__.py

@@ -42,13 +42,23 @@ COMMAND_GROUPS = {
             "name": "Load, save, create & delete data",
             "commands": ["load", "save", "create", "delete"],
         },
+        {
+            "name": "Tracking within shell scripts",
+            "commands": ["track", "finish"],
+        },
         {
             "name": "Describe, annotate & list data",
             "commands": ["describe", "annotate", "list"],
         },
         {
             "name": "Configure",
-            "commands": ["checkout", "switch", "cache", "settings", "migrate"],
+            "commands": [
+                "checkout",
+                "switch",
+                "cache",
+                "settings",
+                "migrate",
+            ],
         },
         {
             "name": "Auth",
@@ -103,6 +113,7 @@ else:
 from lamindb_setup._silence_loggers import silence_loggers
 
 from lamin_cli._cache import cache
+from lamin_cli._io import io
 from lamin_cli._migration import migrate
 from lamin_cli._settings import settings
 
@@ -294,54 +305,60 @@ def info(schema: bool):
 @click.option("--name", type=str, default=None)
 @click.option("--uid", type=str, default=None)
 @click.option("--slug", type=str, default=None)
+@click.option("--permanent", is_flag=True, default=None, help="Permanently delete the entity where applicable, e.g., for artifact, transform, collection.")
 @click.option("--force", is_flag=True, default=False, help="Do not ask for confirmation (only relevant for instance).")
 # fmt: on
-def delete(entity: str, name: str | None = None, uid: str | None = None, slug: str | None = None, force: bool = False):
+def delete(entity: str, name: str | None = None, uid: str | None = None, slug: str | None = None, permanent: bool | None = None, force: bool = False):
     """Delete an entity.
 
     Currently supported: `branch`, `artifact`, `transform`, `collection`, and `instance`. For example:
 
     ```
     lamin delete https://lamin.ai/account/instance/artifact/e2G7k9EVul4JbfsEYAy5
+    lamin delete https://lamin.ai/account/instance/artifact/e2G7k9EVul4JbfsEYAy5 --permanent
     lamin delete branch --name my_branch
     lamin delete instance --slug account/name
     ```
     """
     from lamin_cli._delete import delete as delete_
 
-    return delete_(entity=entity, name=name, uid=uid, slug=slug, force=force)
+    return delete_(entity=entity, name=name, uid=uid, slug=slug, permanent=permanent, force=force)
 
 
 @main.command()
-@click.argument("entity", type=str)
+@click.argument("entity", type=str, required=False)
 @click.option("--uid", help="The uid for the entity.")
 @click.option("--key", help="The key for the entity.")
 @click.option(
     "--with-env", is_flag=True, help="Also return the environment for a tranform."
 )
-def load(entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False):
+def load(entity: str | None = None, uid: str | None = None, key: str | None = None, with_env: bool = False):
     """Load a file or folder into the cache or working directory.
 
-    Pass a URL, `artifact`, or `transform`. For example:
+    Pass a URL or `--key`. For example:
 
     ```
     lamin load https://lamin.ai/account/instance/artifact/e2G7k9EVul4JbfsE
-    lamin load artifact --key mydatasets/mytable.parquet
+    lamin load --key mydatasets/mytable.parquet
+    lamin load --key analysis.ipynb
+    lamin load --key myanalyses/analysis.ipynb --with-env
+    ```
+
+    You can also pass a uid and the entity type:
+
+    ```
     lamin load artifact --uid e2G7k9EVul4JbfsE
-    lamin load transform --key analysis.ipynb
     lamin load transform --uid Vul4JbfsEYAy5
-    lamin load transform --uid Vul4JbfsEYAy5 --with-env
     ```
     """
-    is_slug = entity.count("/") == 1
-    if is_slug:
-        from lamindb_setup._connect_instance import _connect_cli
-        # for backward compat and convenience, connect to the instance
-        return _connect_cli(entity)
-    else:
-        from lamin_cli._load import load as load_
-
-        return load_(entity, uid=uid, key=key, with_env=with_env)
+    from lamin_cli._load import load as load_
+    if entity is not None:
+        is_slug = entity.count("/") == 1
+        if is_slug:
+            from lamindb_setup._connect_instance import _connect_cli
+            # for backward compat
+            return _connect_cli(entity)
+    return load_(entity, uid=uid, key=key, with_env=with_env)
 
 
 def _describe(entity: str = "artifact", uid: str | None = None, key: str | None = None):
@@ -423,6 +440,41 @@ def save(path: str, key: str, description: str, stem_uid: str, project: str, spa
     if save_(path=path, key=key, description=description, stem_uid=stem_uid, project=project, space=space, branch=branch, registry=registry) is not None:
         sys.exit(1)
 
+@main.command()
+def track():
+    """Start tracking a run of a shell script.
+
+    This command works like {func}`~lamindb.track()` in a Python session. Here is an example script:
+
+    ```
+    # my_script.sh
+    set -e         # exit on error
+    lamin track    # initiate a tracked shell script run
+    lamin load --key raw/file1.txt
+    # do something
+    lamin save processed_file1.txt --key processed/file1.txt
+    lamin finish   # mark the shell script run as finished
+    ```
+
+    If you run that script, it will track the run of the script, and save the input and output artifacts:
+
+    ```
+    sh my_script.sh
+    ```
+    """
+    from lamin_cli._context import track as track_
+    return track_()
+
+
+@main.command()
+def finish():
+    """Finish the current tracked run of a shell script.
+
+    This command works like {func}`~lamindb.finish()` in a Python session.
+    """
+    from lamin_cli._context import finish as finish_
+    return finish_()
+
 
 @main.command()
 @click.argument("entity", type=str, default=None, required=False)
@@ -539,6 +591,7 @@ def run(filepath: str, project: str, image_url: str, packages: str, cpu: int, gp
 main.add_command(settings)
 main.add_command(cache)
 main.add_command(migrate)
+main.add_command(io)
 
 # https://stackoverflow.com/questions/57810659/automatically-generate-all-help-documentation-for-click-commands
 # https://claude.ai/chat/73c28487-bec3-4073-8110-50d1a2dd6b84

lamin_cli-1.11.0/lamin_cli/_context.py

@@ -0,0 +1,76 @@
+import os
+import sys
+from pathlib import Path
+
+import click
+from lamin_utils import logger
+from lamindb_setup.core._settings_store import settings_dir
+
+
+def get_current_run_file() -> Path:
+    """Get the path to the file storing the current run UID."""
+    return settings_dir / "current_shell_run.txt"
+
+
+def is_interactive_shell() -> bool:
+    """Check if running in an interactive terminal."""
+    return sys.stdin.isatty() and sys.stdout.isatty() and os.isatty(0)
+
+
+def get_script_filename() -> Path:
+    """Try to get the filename of the calling shell script."""
+    import psutil
+
+    parent = psutil.Process(os.getppid())
+    cmdline = parent.cmdline()
+
+    # For shells like bash, sh, zsh
+    if parent.name() in ["bash", "sh", "zsh", "dash"]:
+        # cmdline is typically: ['/bin/bash', 'script.sh', ...]
+        if len(cmdline) > 1 and not cmdline[1].startswith("-"):
+            return Path(cmdline[1])
+    raise click.ClickException(
+        "Cannot determine script filename. Please run in an interactive shell."
+    )
+
+
+def track():
+    import lamindb as ln
+
+    if not ln.setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+    path = get_script_filename()
+    source_code = path.read_text()
+    transform = ln.Transform(
+        key=path.name, source_code=source_code, type="script"
+    ).save()
+    run = ln.Run(transform=transform).save()
+    current_run_file = get_current_run_file()
+    current_run_file.parent.mkdir(parents=True, exist_ok=True)
+    current_run_file.write_text(run.uid)
+    logger.important(f"started tracking shell run: {run.uid}")
+
+
+def finish():
+    from datetime import datetime, timezone
+
+    import lamindb as ln
+
+    if not ln.setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    current_run_file = get_current_run_file()
+    if not current_run_file.exists():
+        raise click.ClickException(
+            "No active run to finish. Please run `lamin track` first."
+        )
+    run = ln.Run.get(uid=current_run_file.read_text().strip())
+    run._status_code = 0
+    run.finished_at = datetime.now(timezone.utc)
+    run.save()
+    current_run_file.unlink()
+    logger.important(f"finished tracking shell run: {run.uid}")

{lamin_cli-1.9.0 → lamin_cli-1.11.0}/lamin_cli/_delete.py

@@ -9,6 +9,7 @@ def delete(
     name: str | None = None,
     uid: str | None = None,
     slug: str | None = None,
+    permanent: bool | None = None,
     force: bool = False,
 ):
     # TODO: refactor to abstract getting and deleting across entities
@@ -21,22 +22,22 @@ def delete(
         assert name is not None, "You have to pass a name for deleting a branch."
         from lamindb import Branch
 
-        Branch.get(name=name).delete()
+        Branch.get(name=name).delete(permanent=permanent)
     elif entity == "artifact":
         assert uid is not None, "You have to pass a uid for deleting an artifact."
         from lamindb import Artifact
 
-        Artifact.get(uid).delete()
+        Artifact.get(uid).delete(permanent=permanent)
     elif entity == "transform":
         assert uid is not None, "You have to pass a uid for deleting an transform."
         from lamindb import Transform
 
-        Transform.get(uid).delete()
+        Transform.get(uid).delete(permanent=permanent)
     elif entity == "collection":
         assert uid is not None, "You have to pass a uid for deleting an collection."
         from lamindb import Collection
 
-        Collection.get(uid).delete()
+        Collection.get(uid).delete(permanent=permanent)
     elif entity == "instance":
         return delete_instance(slug, force=force)
     else:  # backwards compatibility

lamin_cli-1.11.0/lamin_cli/_io.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import lamindb_setup as ln_setup
+
+from lamin_cli.clone._clone_verification import (
+    _count_instance_records,
+)
+
+if os.environ.get("NO_RICH"):
+    import click as click
+else:
+    import rich_click as click
+
+
+@click.group()
+def io():
+    """Import and export instances."""
+
+
+# fmt: off
+@io.command("snapshot")
+@click.option("--upload/--no-upload", is_flag=True, help="Whether to upload the snapshot.", default=True)
+@click.option("--track/--no-track", is_flag=True, help="Whether to track snapshot generation.", default=True)
+# fmt: on
+def snapshot(upload: bool, track: bool) -> None:
+    """Create a SQLite snapshot of the connected instance."""
+    if not ln_setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    instance_owner = ln_setup.settings.instance.owner
+    instance_name = ln_setup.settings.instance.name
+
+    ln_setup.connect(f"{instance_owner}/{instance_name}", use_root_db_user=True)
+
+    import lamindb as ln
+
+    original_counts = _count_instance_records()
+
+    modules_without_lamindb = ln_setup.settings.instance.modules
+    modules_complete = modules_without_lamindb.copy()
+    modules_complete.add("lamindb")
+
+
+    with tempfile.TemporaryDirectory() as export_dir:
+        if track:
+            ln.track()
+        ln_setup.io.export_db(module_names=modules_complete, output_dir=export_dir)
+        if track:
+            ln.finish()
+
+        script_path = (
+            Path(__file__).parent / "clone" / "create_sqlite_clone_and_import_db.py"
+        )
+        result = subprocess.run(
+            [
+                sys.executable,
+                str(script_path),
+                "--instance-name",
+                instance_name,
+                "--export-dir",
+                export_dir,
+                "--modules",
+                ",".join(modules_without_lamindb),
+                "--original-counts",
+                json.dumps(original_counts),
+            ],
+            check=False,
+            stderr=subprocess.PIPE,
+            text=True,
+            cwd=Path.cwd(),
+        )
+        if result.returncode != 0:
+            try:
+                mismatches = json.loads(result.stderr.strip())
+                error_msg = "Record count mismatch detected:\n" + "\n".join(
+                    [f"  {table}: original={orig}, clone={clone}"
+                    for table, (orig, clone) in mismatches.items()]
+                )
+                raise click.ClickException(error_msg)
+            except (json.JSONDecodeError, AttributeError, ValueError, TypeError):
+                raise click.ClickException(f"Clone verification failed:\n{result.stderr}") from None
+
+
+        ln_setup.connect(f"{instance_owner}/{instance_name}", use_root_db_user=True)
+        if upload:
+            ln_setup.core._clone.upload_sqlite_clone(
+                local_sqlite_path=f"{instance_name}-clone/.lamindb/lamin.db",
+                compress=True,
+            )
+
+        ln_setup.disconnect()
+
+
+# fmt: off
+@io.command("exportdb")
+@click.option("--modules", type=str, default=None, help="Comma-separated list of modules to export (e.g., 'lamindb,bionty').",)
+@click.option("--output-dir", type=str, help="Output directory for exported parquet files.")
+@click.option("--max-workers", type=int, default=8, help="Number of parallel workers.")
+@click.option("--chunk-size", type=int, default=500_000, help="Number of rows per chunk for large tables.")
+# fmt: on
+def exportdb(modules: str | None, output_dir: str, max_workers: int, chunk_size: int):
+    """Export registry tables to parquet files."""
+    if not ln_setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    module_list = modules.split(",") if modules else None
+    ln_setup.io.export_db(
+        module_names=module_list,
+        output_dir=output_dir,
+        max_workers=max_workers,
+        chunk_size=chunk_size,
+    )
+
+
+# fmt: off
+@io.command("importdb")
+@click.option("--modules", type=str, default=None, help="Comma-separated list of modules to import (e.g., 'lamindb,bionty').")
+@click.option("--input-dir", type=str, help="Input directory containing exported parquet files.")
+@click.option("--if-exists", type=click.Choice(["fail", "replace", "append"]), default="replace", help="How to handle existing data.")
+# fmt: on
+def importdb(modules: str | None, input_dir: str, if_exists: str):
+    """Import registry tables from parquet files."""
+    if not ln_setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    module_list = modules.split(",") if modules else None
+    ln_setup.io.import_db(
+        module_names=module_list,
+        input_dir=input_dir,
+        if_exists=if_exists,
+    )

{lamin_cli-1.9.0 → lamin_cli-1.11.0}/lamin_cli/_load.py

@@ -6,15 +6,36 @@ from pathlib import Path
 
 from lamin_utils import logger
 
-from ._save import parse_title_r_notebook
+from ._context import get_current_run_file
+from ._save import infer_registry_from_path, parse_title_r_notebook
 from .urls import decompose_url
 
 
 def load(
-    entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False
+    entity: str | None = None,
+    uid: str | None = None,
+    key: str | None = None,
+    with_env: bool = False,
 ):
+    """Load artifact, collection, or transform from LaminDB.
+
+    Args:
+        entity: URL containing 'lamin', or 'artifact', 'collection', or 'transform'
+        uid: Unique identifier (prefix matching supported)
+        key: Key identifier
+        with_env: If True, also load environment requirements file for transforms
+
+    Returns:
+        Path to loaded transform, or None for artifacts/collections
+    """
     import lamindb_setup as ln_setup
 
+    if entity is None:
+        if key is None:
+            raise SystemExit("Either entity or key has to be provided.")
+        else:
+            entity = infer_registry_from_path(key)
+
     if entity.startswith("https://") and "lamin" in entity:
         url = entity
         instance, entity, uid = decompose_url(url)
@@ -28,6 +49,10 @@ def load(
     ln_setup.connect(instance)
     import lamindb as ln
 
+    current_run = None
+    if get_current_run_file().exists():
+        current_run = ln.Run.get(uid=get_current_run_file().read_text().strip())
+
     def script_to_notebook(
         transform: ln.Transform, notebook_path: Path, bump_revision: bool = False
     ) -> None:
@@ -102,27 +127,25 @@ def load(
                 transforms = transforms.order_by("-created_at")
             transform = transforms.first()
 
-            target_relpath = Path(transform.key)
-            if len(target_relpath.parents) > 1:
-                logger.important(
-                    "preserve the folder structure for versioning:"
-                    f" {target_relpath.parent}/"
-                )
-                target_relpath.parent.mkdir(parents=True, exist_ok=True)
-            if target_relpath.exists():
-                response = input(f"! {target_relpath} exists: replace? (y/n)")
+            target_path = Path(transform.key)
+            if ln_setup.settings.dev_dir is not None:
+                target_path = ln_setup.settings.dev_dir / target_path
+            if len(target_path.parents) > 1:
+                target_path.parent.mkdir(parents=True, exist_ok=True)
+            if target_path.exists():
+                response = input(f"! {target_path} exists: replace? (y/n)")
                 if response != "y":
                     raise SystemExit("Aborted.")
 
             if transform.source_code is not None:
-                if target_relpath.suffix in (".ipynb", ".Rmd", ".qmd"):
-                    script_to_notebook(transform, target_relpath, bump_revision=True)
+                if target_path.suffix in (".ipynb", ".Rmd", ".qmd"):
+                    script_to_notebook(transform, target_path, bump_revision=True)
                 else:
-                    target_relpath.write_text(transform.source_code)
+                    target_path.write_text(transform.source_code)
             else:
                 raise SystemExit("No source code available for this transform.")
 
-            logger.important(f"{transform.type} is here: {target_relpath}")
+            logger.important(f"{transform.type} is here: {target_path}")
 
             if with_env:
                 ln.settings.track_run_inputs = False
@@ -132,8 +155,7 @@ def load(
                 ):
                     filepath_env_cache = transform.latest_run.environment.cache()
                     target_env_filename = (
-                        target_relpath.parent
-                        / f"{target_relpath.stem}__requirements.txt"
+                        target_path.parent / f"{target_path.stem}__requirements.txt"
                     )
                     shutil.move(filepath_env_cache, target_env_filename)
                     logger.important(f"environment is here: {target_env_filename}")
@@ -142,14 +164,13 @@ def load(
                         "latest transform run with environment doesn't exist"
                     )
 
-            return target_relpath
+            return target_path
         case "artifact" | "collection":
             ln.settings.track_run_inputs = False
 
             EntityClass = ln.Artifact if entity == "artifact" else ln.Collection
 
-            # we don't use .get here because DoesNotExist is hard to catch
-            # due to private django API
+            # we don't use .get here because DoesNotExist is hard to catch due to private django API
             # we use `.objects` here because we don't want to exclude kind = __lamindb_run__ artifacts
             if query_by_uid:
                 entities = EntityClass.objects.filter(uid__startswith=uid)
@@ -166,7 +187,7 @@ def load(
                 entities = entities.order_by("-created_at")
 
             entity_obj = entities.first()
-            cache_path = entity_obj.cache()
+            cache_path = entity_obj.cache(is_run_input=current_run)
 
             # collection gives us a list of paths
             if isinstance(cache_path, list):

{lamin_cli-1.9.0 → lamin_cli-1.11.0}/lamin_cli/_migration.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import os
-from typing import Optional
 
 if os.environ.get("NO_RICH"):
     import click as click