lamin_cli 1.9.0__py2.py3-none-any.whl → 1.11.0__py2.py3-none-any.whl

lamin_cli/__init__.py

@@ -2,10 +2,11 @@
 
 This is the command line interface for interacting with LaminDB & LaminHub.
 
-The interface is defined in `__main__.py`. The root API here is used by LaminR to replicate the CLI functionality.
+The interface is defined in `__main__.py`.
+The root API here is used by LaminR to replicate the CLI functionality.
 """
 
-__version__ = "1.9.0"
+__version__ = "1.11.0"
 
 from lamindb_setup import disconnect, logout
 from lamindb_setup._connect_instance import _connect_cli as connect
@@ -15,4 +16,12 @@ from lamindb_setup._setup_user import login
 from ._delete import delete
 from ._save import save
 
-__all__ = ["save", "init", "connect", "delete", "login", "disconnect"]
+__all__ = [
+    "save",
+    "init",
+    "connect",
+    "delete",
+    "login",
+    "logout",
+    "disconnect",
+]

lamin_cli/__main__.py

@@ -42,13 +42,23 @@ COMMAND_GROUPS = {
             "name": "Load, save, create & delete data",
             "commands": ["load", "save", "create", "delete"],
         },
+        {
+            "name": "Tracking within shell scripts",
+            "commands": ["track", "finish"],
+        },
         {
             "name": "Describe, annotate & list data",
             "commands": ["describe", "annotate", "list"],
         },
         {
             "name": "Configure",
-            "commands": ["checkout", "switch", "cache", "settings", "migrate"],
+            "commands": [
+                "checkout",
+                "switch",
+                "cache",
+                "settings",
+                "migrate",
+            ],
         },
         {
             "name": "Auth",
@@ -103,6 +113,7 @@ else:
 from lamindb_setup._silence_loggers import silence_loggers
 
 from lamin_cli._cache import cache
+from lamin_cli._io import io
 from lamin_cli._migration import migrate
 from lamin_cli._settings import settings
 
@@ -294,54 +305,60 @@ def info(schema: bool):
 @click.option("--name", type=str, default=None)
 @click.option("--uid", type=str, default=None)
 @click.option("--slug", type=str, default=None)
+@click.option("--permanent", is_flag=True, default=None, help="Permanently delete the entity where applicable, e.g., for artifact, transform, collection.")
 @click.option("--force", is_flag=True, default=False, help="Do not ask for confirmation (only relevant for instance).")
 # fmt: on
-def delete(entity: str, name: str | None = None, uid: str | None = None, slug: str | None = None, force: bool = False):
+def delete(entity: str, name: str | None = None, uid: str | None = None, slug: str | None = None, permanent: bool | None = None, force: bool = False):
     """Delete an entity.
 
     Currently supported: `branch`, `artifact`, `transform`, `collection`, and `instance`. For example:
 
     ```
     lamin delete https://lamin.ai/account/instance/artifact/e2G7k9EVul4JbfsEYAy5
+    lamin delete https://lamin.ai/account/instance/artifact/e2G7k9EVul4JbfsEYAy5 --permanent
     lamin delete branch --name my_branch
     lamin delete instance --slug account/name
     ```
     """
     from lamin_cli._delete import delete as delete_
 
-    return delete_(entity=entity, name=name, uid=uid, slug=slug, force=force)
+    return delete_(entity=entity, name=name, uid=uid, slug=slug, permanent=permanent, force=force)
 
 
 @main.command()
-@click.argument("entity", type=str)
+@click.argument("entity", type=str, required=False)
 @click.option("--uid", help="The uid for the entity.")
 @click.option("--key", help="The key for the entity.")
 @click.option(
     "--with-env", is_flag=True, help="Also return the environment for a tranform."
 )
-def load(entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False):
+def load(entity: str | None = None, uid: str | None = None, key: str | None = None, with_env: bool = False):
     """Load a file or folder into the cache or working directory.
 
-    Pass a URL, `artifact`, or `transform`. For example:
+    Pass a URL or `--key`. For example:
 
     ```
     lamin load https://lamin.ai/account/instance/artifact/e2G7k9EVul4JbfsE
-    lamin load artifact --key mydatasets/mytable.parquet
+    lamin load --key mydatasets/mytable.parquet
+    lamin load --key analysis.ipynb
+    lamin load --key myanalyses/analysis.ipynb --with-env
+    ```
+
+    You can also pass a uid and the entity type:
+
+    ```
     lamin load artifact --uid e2G7k9EVul4JbfsE
-    lamin load transform --key analysis.ipynb
     lamin load transform --uid Vul4JbfsEYAy5
-    lamin load transform --uid Vul4JbfsEYAy5 --with-env
     ```
     """
-    is_slug = entity.count("/") == 1
-    if is_slug:
-        from lamindb_setup._connect_instance import _connect_cli
-        # for backward compat and convenience, connect to the instance
-        return _connect_cli(entity)
-    else:
-        from lamin_cli._load import load as load_
-
-        return load_(entity, uid=uid, key=key, with_env=with_env)
+    from lamin_cli._load import load as load_
+    if entity is not None:
+        is_slug = entity.count("/") == 1
+        if is_slug:
+            from lamindb_setup._connect_instance import _connect_cli
+            # for backward compat
+            return _connect_cli(entity)
+    return load_(entity, uid=uid, key=key, with_env=with_env)
 
 
 def _describe(entity: str = "artifact", uid: str | None = None, key: str | None = None):
@@ -423,6 +440,41 @@ def save(path: str, key: str, description: str, stem_uid: str, project: str, spa
     if save_(path=path, key=key, description=description, stem_uid=stem_uid, project=project, space=space, branch=branch, registry=registry) is not None:
         sys.exit(1)
 
+@main.command()
+def track():
+    """Start tracking a run of a shell script.
+
+    This command works like {func}`~lamindb.track()` in a Python session. Here is an example script:
+
+    ```
+    # my_script.sh
+    set -e         # exit on error
+    lamin track    # initiate a tracked shell script run
+    lamin load --key raw/file1.txt
+    # do something
+    lamin save processed_file1.txt --key processed/file1.txt
+    lamin finish   # mark the shell script run as finished
+    ```
+
+    If you run that script, it will track the run of the script, and save the input and output artifacts:
+
+    ```
+    sh my_script.sh
+    ```
+    """
+    from lamin_cli._context import track as track_
+    return track_()
+
+
+@main.command()
+def finish():
+    """Finish the current tracked run of a shell script.
+
+    This command works like {func}`~lamindb.finish()` in a Python session.
+    """
+    from lamin_cli._context import finish as finish_
+    return finish_()
+
 
 @main.command()
 @click.argument("entity", type=str, default=None, required=False)
@@ -539,6 +591,7 @@ def run(filepath: str, project: str, image_url: str, packages: str, cpu: int, gp
 main.add_command(settings)
 main.add_command(cache)
 main.add_command(migrate)
+main.add_command(io)
 
 # https://stackoverflow.com/questions/57810659/automatically-generate-all-help-documentation-for-click-commands
 # https://claude.ai/chat/73c28487-bec3-4073-8110-50d1a2dd6b84

lamin_cli/_context.py

@@ -0,0 +1,76 @@
+import os
+import sys
+from pathlib import Path
+
+import click
+from lamin_utils import logger
+from lamindb_setup.core._settings_store import settings_dir
+
+
+def get_current_run_file() -> Path:
+    """Get the path to the file storing the current run UID."""
+    return settings_dir / "current_shell_run.txt"
+
+
+def is_interactive_shell() -> bool:
+    """Check if running in an interactive terminal."""
+    return sys.stdin.isatty() and sys.stdout.isatty() and os.isatty(0)
+
+
+def get_script_filename() -> Path:
+    """Try to get the filename of the calling shell script."""
+    import psutil
+
+    parent = psutil.Process(os.getppid())
+    cmdline = parent.cmdline()
+
+    # For shells like bash, sh, zsh
+    if parent.name() in ["bash", "sh", "zsh", "dash"]:
+        # cmdline is typically: ['/bin/bash', 'script.sh', ...]
+        if len(cmdline) > 1 and not cmdline[1].startswith("-"):
+            return Path(cmdline[1])
+    raise click.ClickException(
+        "Cannot determine script filename. Please run in an interactive shell."
+    )
+
+
+def track():
+    import lamindb as ln
+
+    if not ln.setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+    path = get_script_filename()
+    source_code = path.read_text()
+    transform = ln.Transform(
+        key=path.name, source_code=source_code, type="script"
+    ).save()
+    run = ln.Run(transform=transform).save()
+    current_run_file = get_current_run_file()
+    current_run_file.parent.mkdir(parents=True, exist_ok=True)
+    current_run_file.write_text(run.uid)
+    logger.important(f"started tracking shell run: {run.uid}")
+
+
+def finish():
+    from datetime import datetime, timezone
+
+    import lamindb as ln
+
+    if not ln.setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    current_run_file = get_current_run_file()
+    if not current_run_file.exists():
+        raise click.ClickException(
+            "No active run to finish. Please run `lamin track` first."
+        )
+    run = ln.Run.get(uid=current_run_file.read_text().strip())
+    run._status_code = 0
+    run.finished_at = datetime.now(timezone.utc)
+    run.save()
+    current_run_file.unlink()
+    logger.important(f"finished tracking shell run: {run.uid}")

lamin_cli/_delete.py

@@ -9,6 +9,7 @@ def delete(
     name: str | None = None,
     uid: str | None = None,
     slug: str | None = None,
+    permanent: bool | None = None,
     force: bool = False,
 ):
     # TODO: refactor to abstract getting and deleting across entities
@@ -21,22 +22,22 @@ def delete(
         assert name is not None, "You have to pass a name for deleting a branch."
         from lamindb import Branch
 
-        Branch.get(name=name).delete()
+        Branch.get(name=name).delete(permanent=permanent)
     elif entity == "artifact":
         assert uid is not None, "You have to pass a uid for deleting an artifact."
         from lamindb import Artifact
 
-        Artifact.get(uid).delete()
+        Artifact.get(uid).delete(permanent=permanent)
     elif entity == "transform":
         assert uid is not None, "You have to pass a uid for deleting an transform."
         from lamindb import Transform
 
-        Transform.get(uid).delete()
+        Transform.get(uid).delete(permanent=permanent)
     elif entity == "collection":
         assert uid is not None, "You have to pass a uid for deleting an collection."
         from lamindb import Collection
 
-        Collection.get(uid).delete()
+        Collection.get(uid).delete(permanent=permanent)
     elif entity == "instance":
         return delete_instance(slug, force=force)
     else:  # backwards compatibility

lamin_cli/_io.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import lamindb_setup as ln_setup
+
+from lamin_cli.clone._clone_verification import (
+    _count_instance_records,
+)
+
+if os.environ.get("NO_RICH"):
+    import click as click
+else:
+    import rich_click as click
+
+
+@click.group()
+def io():
+    """Import and export instances."""
+
+
+# fmt: off
+@io.command("snapshot")
+@click.option("--upload/--no-upload", is_flag=True, help="Whether to upload the snapshot.", default=True)
+@click.option("--track/--no-track", is_flag=True, help="Whether to track snapshot generation.", default=True)
+# fmt: on
+def snapshot(upload: bool, track: bool) -> None:
+    """Create a SQLite snapshot of the connected instance."""
+    if not ln_setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    instance_owner = ln_setup.settings.instance.owner
+    instance_name = ln_setup.settings.instance.name
+
+    ln_setup.connect(f"{instance_owner}/{instance_name}", use_root_db_user=True)
+
+    import lamindb as ln
+
+    original_counts = _count_instance_records()
+
+    modules_without_lamindb = ln_setup.settings.instance.modules
+    modules_complete = modules_without_lamindb.copy()
+    modules_complete.add("lamindb")
+
+
+    with tempfile.TemporaryDirectory() as export_dir:
+        if track:
+            ln.track()
+        ln_setup.io.export_db(module_names=modules_complete, output_dir=export_dir)
+        if track:
+            ln.finish()
+
+        script_path = (
+            Path(__file__).parent / "clone" / "create_sqlite_clone_and_import_db.py"
+        )
+        result = subprocess.run(
+            [
+                sys.executable,
+                str(script_path),
+                "--instance-name",
+                instance_name,
+                "--export-dir",
+                export_dir,
+                "--modules",
+                ",".join(modules_without_lamindb),
+                "--original-counts",
+                json.dumps(original_counts),
+            ],
+            check=False,
+            stderr=subprocess.PIPE,
+            text=True,
+            cwd=Path.cwd(),
+        )
+        if result.returncode != 0:
+            try:
+                mismatches = json.loads(result.stderr.strip())
+                error_msg = "Record count mismatch detected:\n" + "\n".join(
+                    [f"  {table}: original={orig}, clone={clone}"
+                    for table, (orig, clone) in mismatches.items()]
+                )
+                raise click.ClickException(error_msg)
+            except (json.JSONDecodeError, AttributeError, ValueError, TypeError):
+                raise click.ClickException(f"Clone verification failed:\n{result.stderr}") from None
+
+
+        ln_setup.connect(f"{instance_owner}/{instance_name}", use_root_db_user=True)
+        if upload:
+            ln_setup.core._clone.upload_sqlite_clone(
+                local_sqlite_path=f"{instance_name}-clone/.lamindb/lamin.db",
+                compress=True,
+            )
+
+        ln_setup.disconnect()
+
+
+# fmt: off
+@io.command("exportdb")
+@click.option("--modules", type=str, default=None, help="Comma-separated list of modules to export (e.g., 'lamindb,bionty').",)
+@click.option("--output-dir", type=str, help="Output directory for exported parquet files.")
+@click.option("--max-workers", type=int, default=8, help="Number of parallel workers.")
+@click.option("--chunk-size", type=int, default=500_000, help="Number of rows per chunk for large tables.")
+# fmt: on
+def exportdb(modules: str | None, output_dir: str, max_workers: int, chunk_size: int):
+    """Export registry tables to parquet files."""
+    if not ln_setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    module_list = modules.split(",") if modules else None
+    ln_setup.io.export_db(
+        module_names=module_list,
+        output_dir=output_dir,
+        max_workers=max_workers,
+        chunk_size=chunk_size,
+    )
+
+
+# fmt: off
+@io.command("importdb")
+@click.option("--modules", type=str, default=None, help="Comma-separated list of modules to import (e.g., 'lamindb,bionty').")
+@click.option("--input-dir", type=str, help="Input directory containing exported parquet files.")
+@click.option("--if-exists", type=click.Choice(["fail", "replace", "append"]), default="replace", help="How to handle existing data.")
+# fmt: on
+def importdb(modules: str | None, input_dir: str, if_exists: str):
+    """Import registry tables from parquet files."""
+    if not ln_setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    module_list = modules.split(",") if modules else None
+    ln_setup.io.import_db(
+        module_names=module_list,
+        input_dir=input_dir,
+        if_exists=if_exists,
+    )

lamin_cli/_load.py

@@ -6,15 +6,36 @@ from pathlib import Path
 
 from lamin_utils import logger
 
-from ._save import parse_title_r_notebook
+from ._context import get_current_run_file
+from ._save import infer_registry_from_path, parse_title_r_notebook
 from .urls import decompose_url
 
 
 def load(
-    entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False
+    entity: str | None = None,
+    uid: str | None = None,
+    key: str | None = None,
+    with_env: bool = False,
 ):
+    """Load artifact, collection, or transform from LaminDB.
+
+    Args:
+        entity: URL containing 'lamin', or 'artifact', 'collection', or 'transform'
+        uid: Unique identifier (prefix matching supported)
+        key: Key identifier
+        with_env: If True, also load environment requirements file for transforms
+
+    Returns:
+        Path to loaded transform, or None for artifacts/collections
+    """
     import lamindb_setup as ln_setup
 
+    if entity is None:
+        if key is None:
+            raise SystemExit("Either entity or key has to be provided.")
+        else:
+            entity = infer_registry_from_path(key)
+
     if entity.startswith("https://") and "lamin" in entity:
         url = entity
         instance, entity, uid = decompose_url(url)
@@ -28,6 +49,10 @@ def load(
     ln_setup.connect(instance)
     import lamindb as ln
 
+    current_run = None
+    if get_current_run_file().exists():
+        current_run = ln.Run.get(uid=get_current_run_file().read_text().strip())
+
     def script_to_notebook(
         transform: ln.Transform, notebook_path: Path, bump_revision: bool = False
     ) -> None:
@@ -102,27 +127,25 @@ def load(
                 transforms = transforms.order_by("-created_at")
             transform = transforms.first()
 
-            target_relpath = Path(transform.key)
-            if len(target_relpath.parents) > 1:
-                logger.important(
-                    "preserve the folder structure for versioning:"
-                    f" {target_relpath.parent}/"
-                )
-                target_relpath.parent.mkdir(parents=True, exist_ok=True)
-            if target_relpath.exists():
-                response = input(f"! {target_relpath} exists: replace? (y/n)")
+            target_path = Path(transform.key)
+            if ln_setup.settings.dev_dir is not None:
+                target_path = ln_setup.settings.dev_dir / target_path
+            if len(target_path.parents) > 1:
+                target_path.parent.mkdir(parents=True, exist_ok=True)
+            if target_path.exists():
+                response = input(f"! {target_path} exists: replace? (y/n)")
                 if response != "y":
                     raise SystemExit("Aborted.")
 
             if transform.source_code is not None:
-                if target_relpath.suffix in (".ipynb", ".Rmd", ".qmd"):
-                    script_to_notebook(transform, target_relpath, bump_revision=True)
+                if target_path.suffix in (".ipynb", ".Rmd", ".qmd"):
+                    script_to_notebook(transform, target_path, bump_revision=True)
                 else:
-                    target_relpath.write_text(transform.source_code)
+                    target_path.write_text(transform.source_code)
             else:
                 raise SystemExit("No source code available for this transform.")
 
-            logger.important(f"{transform.type} is here: {target_relpath}")
+            logger.important(f"{transform.type} is here: {target_path}")
 
             if with_env:
                 ln.settings.track_run_inputs = False
@@ -132,8 +155,7 @@ def load(
                 ):
                     filepath_env_cache = transform.latest_run.environment.cache()
                     target_env_filename = (
-                        target_relpath.parent
-                        / f"{target_relpath.stem}__requirements.txt"
+                        target_path.parent / f"{target_path.stem}__requirements.txt"
                     )
                     shutil.move(filepath_env_cache, target_env_filename)
                     logger.important(f"environment is here: {target_env_filename}")
@@ -142,14 +164,13 @@ def load(
                         "latest transform run with environment doesn't exist"
                     )
 
-            return target_relpath
+            return target_path
         case "artifact" | "collection":
             ln.settings.track_run_inputs = False
 
             EntityClass = ln.Artifact if entity == "artifact" else ln.Collection
 
-            # we don't use .get here because DoesNotExist is hard to catch
-            # due to private django API
+            # we don't use .get here because DoesNotExist is hard to catch due to private django API
             # we use `.objects` here because we don't want to exclude kind = __lamindb_run__ artifacts
             if query_by_uid:
                 entities = EntityClass.objects.filter(uid__startswith=uid)
@@ -166,7 +187,7 @@ def load(
                 entities = entities.order_by("-created_at")
 
             entity_obj = entities.first()
-            cache_path = entity_obj.cache()
+            cache_path = entity_obj.cache(is_run_input=current_run)
 
             # collection gives us a list of paths
             if isinstance(cache_path, list):

lamin_cli/_migration.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import os
-from typing import Optional
 
 if os.environ.get("NO_RICH"):
     import click as click

lamin_cli/_save.py

@@ -2,25 +2,30 @@ from __future__ import annotations
 
 import os
 import re
-import sys
 from pathlib import Path
-from typing import TYPE_CHECKING
 
 import click
+import lamindb_setup as ln_setup
 from lamin_utils import logger
 from lamindb_setup.core.hashing import hash_file
 
+from lamin_cli._context import get_current_run_file
+
 
 def infer_registry_from_path(path: Path | str) -> str:
     suffixes_transform = {
         "py": {".py", ".ipynb"},
         "R": {".R", ".qmd", ".Rmd"},
+        "sh": {".sh"},
     }
     if isinstance(path, str):
         path = Path(path)
     registry = (
         "transform"
-        if path.suffix in suffixes_transform["py"].union(suffixes_transform["R"])
+        if path.suffix
+        in suffixes_transform["py"]
+        .union(suffixes_transform["R"])
+        .union(suffixes_transform["sh"])
         else "artifact"
     )
     return registry
@@ -43,9 +48,11 @@ def parse_uid_from_code(content: str, suffix: str) -> str | None:
             r'track\(\s*(?:transform\s*=\s*)?([\'"])([a-zA-Z0-9]{12,16})\1'
         )
         uid_pattern = None
+    elif suffix == ".sh":
+        return None
     else:
         raise SystemExit(
-            "Only .py, .ipynb, .R, .qmd, .Rmd files are supported for saving"
+            "Only .py, .ipynb, .R, .qmd, .Rmd, .sh files are supported for saving"
             " transforms."
         )
 
@@ -83,18 +90,23 @@ def save(
 ) -> str | None:
     import lamindb as ln
     from lamindb._finish import save_context_core
+    from lamindb_setup.core._settings_store import settings_dir
     from lamindb_setup.core.upath import LocalPathClasses, UPath, create_path
 
+    current_run = None
+    if get_current_run_file().exists():
+        current_run = ln.Run.get(uid=get_current_run_file().read_text().strip())
+
     # this allows to have the correct treatment of credentials in case of cloud paths
-    path = create_path(path)
+    ppath = create_path(path)
     # isinstance is needed to cast the type of path to UPath
     # to avoid mypy erors
-    assert isinstance(path, UPath)
-    if not path.exists():
-        raise click.BadParameter(f"Path {path} does not exist", param_hint="path")
+    assert isinstance(ppath, UPath)
+    if not ppath.exists():
+        raise click.BadParameter(f"Path {ppath} does not exist", param_hint="path")
 
     if registry is None:
-        registry = infer_registry_from_path(path)
+        registry = infer_registry_from_path(ppath)
 
     if project is not None:
         project_record = ln.Project.filter(
@@ -121,7 +133,7 @@ def save(
                 f"Branch '{branch}' not found, either create it with `ln.Branch(name='...').save()` or fix typos."
             )
 
-    is_cloud_path = not isinstance(path, LocalPathClasses)
+    is_cloud_path = not isinstance(ppath, LocalPathClasses)
 
     if registry == "artifact":
         ln.settings.creation.artifact_silence_missing_run_warning = True
@@ -144,12 +156,13 @@ def save(
             return "missing-key-or-description"
 
         artifact = ln.Artifact(
-            path,
+            ppath,
             key=key,
             description=description,
             revises=revises,
             branch=branch_record,
             space=space_record,
+            run=current_run,
         ).save()
         logger.important(f"saved: {artifact}")
         logger.important(f"storage path: {artifact.path}")
@@ -167,34 +180,40 @@ def save(
     if registry == "transform":
         if key is not None:
             logger.warning(
-                "key is ignored for transforms, the transform key is determined by the filename"
+                "key is ignored for transforms, the transform key is determined by the filename and the development directory (dev-dir)"
             )
         if is_cloud_path:
             logger.error("Can not register a transform from a cloud path")
             return "transform-with-cloud-path"
 
-        if path.suffix in {".qmd", ".Rmd"}:
-            html_file_exists = path.with_suffix(".html").exists()
-            nb_html_file_exists = path.with_suffix(".nb.html").exists()
+        if ppath.suffix in {".qmd", ".Rmd"}:
+            html_file_exists = ppath.with_suffix(".html").exists()
+            nb_html_file_exists = ppath.with_suffix(".nb.html").exists()
 
             if not html_file_exists and not nb_html_file_exists:
                 logger.error(
-                    f"Please export your {path.suffix} file as an html file here"
-                    f" {path.with_suffix('.html')}"
+                    f"Please export your {ppath.suffix} file as an html file here"
+                    f" {ppath.with_suffix('.html')}"
                 )
                 return "export-qmd-Rmd-as-html"
             elif html_file_exists and nb_html_file_exists:
                 logger.error(
-                    f"Please delete one of\n - {path.with_suffix('.html')}\n -"
-                    f" {path.with_suffix('.nb.html')}"
+                    f"Please delete one of\n - {ppath.with_suffix('.html')}\n -"
+                    f" {ppath.with_suffix('.nb.html')}"
                 )
                 return "delete-html-or-nb-html"
 
-        content = path.read_text()
-        uid = parse_uid_from_code(content, path.suffix)
+        content = ppath.read_text()
+        uid = parse_uid_from_code(content, ppath.suffix)
+
+        ppath = ppath.resolve().expanduser()
+        if ln_setup.settings.dev_dir is not None:
+            key = ppath.relative_to(ln_setup.settings.dev_dir).as_posix()
+        else:
+            key = ppath.name
 
         if uid is not None:
-            logger.important(f"mapped '{path.name}' on uid '{uid}'")
+            logger.important(f"mapped '{ppath.name}' on uid '{uid}'")
             if len(uid) == 16:
                 # is full uid
                 transform = ln.Transform.filter(uid=uid).one_or_none()
@@ -214,12 +233,16 @@ def save(
                 if transform is None:
                     uid = f"{stem_uid}0000"
         else:
-            # TODO: account for folders as we do in ln.track()
-            transform_hash, _ = hash_file(path)
-            transform = ln.Transform.filter(key=path.name, is_latest=True).one_or_none()
+            _, transform_hash, _ = hash_file(ppath)
+            transform = ln.Transform.filter(hash=transform_hash).first()
             if transform is not None and transform.hash is not None:
                 if transform.hash == transform_hash:
                     if transform.type != "notebook":
+                        logger.important(f"transform already saved: {transform}")
+                        if transform.key != key:
+                            transform.key = key
+                            logger.important(f"updated key to '{key}'")
+                            transform.save()
                         return None
                     if os.getenv("LAMIN_TESTING") == "true":
                         response = "y"
@@ -245,21 +268,21 @@ def save(
             if revises is None:
                 raise ln.errors.InvalidArgument("The stem uid is not found.")
         if transform is None:
-            if path.suffix == ".ipynb":
+            if ppath.suffix == ".ipynb":
                 from nbproject.dev import read_notebook
                 from nbproject.dev._meta_live import get_title
 
-                nb = read_notebook(path)
+                nb = read_notebook(ppath)
                 description = get_title(nb)
-            elif path.suffix in {".qmd", ".Rmd"}:
+            elif ppath.suffix in {".qmd", ".Rmd"}:
                 description = parse_title_r_notebook(content)
             else:
                 description = None
             transform = ln.Transform(
                 uid=uid,
                 description=description,
-                key=path.name,
-                type="script" if path.suffix in {".R", ".py"} else "notebook",
+                key=key,
+                type="script" if ppath.suffix in {".R", ".py", ".sh"} else "notebook",
                 revises=revises,
             )
             if space is not None:
@@ -267,7 +290,9 @@ def save(
             if branch is not None:
                 transform.branch = branch_record
             transform.save()
-            logger.important(f"created Transform('{transform.uid}')")
+            logger.important(
+                f"created Transform('{transform.uid}', key='{transform.key}')"
+            )
         if project is not None:
             transform.projects.add(project_record)
             logger.important(f"labeled with project: {project_record.name}")
@@ -292,7 +317,7 @@ def save(
         return_code = save_context_core(
             run=run,
             transform=transform,
-            filepath=path,
+            filepath=ppath,
             from_cli=True,
         )
         return return_code

lamin_cli/clone/_clone_verification.py

@@ -0,0 +1,56 @@
+from concurrent.futures import ThreadPoolExecutor
+
+from django.db import OperationalError, ProgrammingError
+from lamin_utils import logger
+
+
+def _count_instance_records() -> dict[str, int]:
+    """Count all records across SQLRecord registries in parallel.
+
+    Returns:
+        Dictionary mapping table names (format: "app_label.ModelName") to  their record counts.
+
+    Example:
+        >>> counts = _count_all_records()
+        >>> counts
+        {'lamindb.Artifact': 1523, 'lamindb.Collection': 42, 'bionty.Gene': 60000}
+    """
+    # Import here to ensure that models are loaded
+    from django.apps import apps
+    from lamindb.models import SQLRecord
+
+    def _count_model(model):
+        """Count records for a single model."""
+        table_name = f"{model._meta.app_label}.{model.__name__}"
+        try:
+            return (table_name, model.objects.count())
+        except (OperationalError, ProgrammingError) as e:
+            logger.warning(f"Could not count {table_name}: {e}")
+            return (table_name, 0)
+
+    models = [m for m in apps.get_models() if issubclass(m, SQLRecord)]
+
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        results = executor.map(_count_model, models)
+
+    return dict(results)
+
+
+def _compare_record_counts(
+    original: dict[str, int], clone: dict[str, int]
+) -> dict[str, tuple[int, int]]:
+    """Compare record counts and return mismatches."""
+    mismatches = {}
+
+    all_tables = set(original.keys()) | set(clone.keys())
+
+    for table in all_tables:
+        orig_count = original.get(table, 0)
+        clone_count = clone.get(table, 0)
+
+        # we allow a difference of 1 because of tracking
+        # new records during the cloning process
+        if abs(clone_count - orig_count) > 1:
+            mismatches[table] = (orig_count, clone_count)
+
+    return mismatches

lamin_cli/clone/create_sqlite_clone_and_import_db.py

@@ -0,0 +1,51 @@
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import lamindb_setup as ln_setup
+
+if __name__ == "__main__":
+    sys.path.insert(0, str(Path(__file__).parent.parent))
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--instance-name", required=True)
+    parser.add_argument("--export-dir", required=True)
+    parser.add_argument("--modules", required=True)
+    parser.add_argument("--original-counts", required=True)
+    args = parser.parse_args()
+
+    instance_name = args.instance_name
+    export_dir = args.export_dir
+    modules_without_lamindb = {m for m in args.modules.split(",") if m}
+    modules_complete = modules_without_lamindb.copy()
+    modules_complete.add("lamindb")
+
+    ln_setup.init(
+        storage=f"{instance_name}-clone", modules=f"{','.join(modules_without_lamindb)}"
+    )
+
+    ln_setup.io.import_db(
+        module_names=list(modules_complete), input_dir=export_dir, if_exists="replace"
+    )
+
+    from django.db import connection
+
+    with connection.cursor() as cursor:
+        cursor.execute("PRAGMA wal_checkpoint(FULL)")
+
+    from lamin_cli.clone._clone_verification import (
+        _compare_record_counts,
+        _count_instance_records,
+    )
+
+    clone_counts = _count_instance_records()
+    original_counts = json.loads(args.original_counts)
+    mismatches = _compare_record_counts(original_counts, clone_counts)
+    if mismatches:
+        print(json.dumps(mismatches), file=sys.stderr)
+        sys.exit(1)
+
+    from django.db import connections
+
+    connections.close_all()

{lamin_cli-1.9.0.dist-info → lamin_cli-1.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamin_cli
-Version: 1.9.0
+Version: 1.11.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

lamin_cli-1.11.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+lamin_cli/__init__.py,sha256=bd6la8_wxlClB7o8h3FNAjQ6z8HPcZ4QqxvVp4Cvycg,605
+lamin_cli/__main__.py,sha256=DEifHuxyyffXHVpBjn9-LVpVUsuSnRhicxTfsuZ0YUw,20791
+lamin_cli/_annotate.py,sha256=ZD76__K-mQt7UpYqyM1I2lKCs-DraTmnkjsByIHmD-g,1839
+lamin_cli/_cache.py,sha256=oplwE8AcS_9PYptQUZxff2qTIdNFS81clGPkJNWk098,800
+lamin_cli/_context.py,sha256=JLhv98isasX_M_o6tIDkeBWcNH8ryRrl3nk2rmwsuD4,2392
+lamin_cli/_delete.py,sha256=1wp8LQdiWHJznRrm4abxEhApB9derBWEm25ufrzjvIg,1531
+lamin_cli/_io.py,sha256=DboDoecZN5OLCHe5vJzOLuwztW8hxbdRffWj1kmJ9WI,5036
+lamin_cli/_load.py,sha256=v7PCB7sYqGs6jF186aN191lTV2Mr5jdayHKSmdgsw_k,8445
+lamin_cli/_migration.py,sha256=w-TLYC2q6sqcfNtIUewqRW_69_xy_71X3eSV7Ud58jk,1239
+lamin_cli/_save.py,sha256=8dM4blz6caKJtbqND-6xJXUMlGIvo2FwvSu1CDcBPig,12517
+lamin_cli/_settings.py,sha256=nqVM8FO9kjL0SXYqP8x-Xfww6Qth4I86HKKeTirsgc4,2657
+lamin_cli/urls.py,sha256=gc72s4SpaAQA8J50CtCIWlr49DWOSL_a6OM9lXfPouM,367
+lamin_cli/clone/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lamin_cli/clone/_clone_verification.py,sha256=8hRFZt_1Z0i2wMqKBkrioq01RJEjwHy9cYmw6LV4PXI,1835
+lamin_cli/clone/create_sqlite_clone_and_import_db.py,sha256=yv2mLkHdRUao_IkZO_4Swu5w4Nieh1gMrBwPbyn4c1o,1544
+lamin_cli/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lamin_cli/compute/modal.py,sha256=QnR7GyyvWWWkLnou95HxS9xxSQfw1k-SiefM_qRVnU0,6010
+lamin_cli-1.11.0.dist-info/entry_points.txt,sha256=Qms85i9cZPlu-U7RnVZhFsF7vJ9gaLZUFkCjcGcXTpg,49
+lamin_cli-1.11.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamin_cli-1.11.0.dist-info/WHEEL,sha256=ssQ84EZ5gH1pCOujd3iW7HClo_O_aDaClUbX4B8bjKY,100
+lamin_cli-1.11.0.dist-info/METADATA,sha256=4TK6hWuABuBxr1PYF_iBpm7KP8xNH7g4LjMWUkHgDKE,338
+lamin_cli-1.11.0.dist-info/RECORD,,

lamin_cli-1.9.0.dist-info/RECORD

@@ -1,17 +0,0 @@
-lamin_cli/__init__.py,sha256=DFl_WGoPWYCrKssL6X5UCL9Via9gGqBVfd0j8UzEsL0,563
-lamin_cli/__main__.py,sha256=zuzvpfWDvrLxvcLNwAyGQHIGXiFvAra4dnPe7Nygu6M,19225
-lamin_cli/_annotate.py,sha256=ZD76__K-mQt7UpYqyM1I2lKCs-DraTmnkjsByIHmD-g,1839
-lamin_cli/_cache.py,sha256=oplwE8AcS_9PYptQUZxff2qTIdNFS81clGPkJNWk098,800
-lamin_cli/_delete.py,sha256=dxItUgf7At247iYGLagmJ2Ccp5nAWgodPL7c7zfZQ_0,1420
-lamin_cli/_load.py,sha256=OgTGqZQtoJ0GYOhuJihz-izt0F4jM4U_nSX_vhPoukg,7698
-lamin_cli/_migration.py,sha256=XUl_L9_3pTTk5jJoBiqbzf0Bd2LdKKtHa1zPZ4Rla5c,1267
-lamin_cli/_save.py,sha256=EPYsRpl0vD_Q52ksqVyP56ZOvPXBjgTFPeJWyGokN8I,11555
-lamin_cli/_settings.py,sha256=nqVM8FO9kjL0SXYqP8x-Xfww6Qth4I86HKKeTirsgc4,2657
-lamin_cli/urls.py,sha256=gc72s4SpaAQA8J50CtCIWlr49DWOSL_a6OM9lXfPouM,367
-lamin_cli/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lamin_cli/compute/modal.py,sha256=QnR7GyyvWWWkLnou95HxS9xxSQfw1k-SiefM_qRVnU0,6010
-lamin_cli-1.9.0.dist-info/entry_points.txt,sha256=Qms85i9cZPlu-U7RnVZhFsF7vJ9gaLZUFkCjcGcXTpg,49
-lamin_cli-1.9.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamin_cli-1.9.0.dist-info/WHEEL,sha256=ssQ84EZ5gH1pCOujd3iW7HClo_O_aDaClUbX4B8bjKY,100
-lamin_cli-1.9.0.dist-info/METADATA,sha256=YhZjJlaC6E_BDVHstvkFya-5p2qEn6lZCrPUzYkN_8w,337
-lamin_cli-1.9.0.dist-info/RECORD,,