lamin_cli 1.8.2__py2.py3-none-any.whl → 1.10.0__py2.py3-none-any.whl

lamin_cli/__init__.py

@@ -2,10 +2,11 @@
 
 This is the command line interface for interacting with LaminDB & LaminHub.
 
-The interface is defined in `__main__.py`. The root API here is used by LaminR to replicate the CLI functionality.
+The interface is defined in `__main__.py`.
+The root API here is used by LaminR to replicate the CLI functionality.
 """
 
-__version__ = "1.8.2"
+__version__ = "1.10.0"
 
 from lamindb_setup import disconnect, logout
 from lamindb_setup._connect_instance import _connect_cli as connect
@@ -15,4 +16,12 @@ from lamindb_setup._setup_user import login
 from ._delete import delete
 from ._save import save
 
-__all__ = ["save", "init", "connect", "delete", "login", "disconnect"]
+__all__ = [
+    "save",
+    "init",
+    "connect",
+    "delete",
+    "login",
+    "logout",
+    "disconnect",
+]

lamin_cli/__main__.py

@@ -48,7 +48,13 @@ COMMAND_GROUPS = {
         },
         {
             "name": "Configure",
-            "commands": ["checkout", "switch", "cache", "settings", "migrate"],
+            "commands": [
+                "checkout",
+                "switch",
+                "cache",
+                "settings",
+                "migrate",
+            ],
         },
         {
             "name": "Auth",
@@ -103,6 +109,7 @@ else:
 from lamindb_setup._silence_loggers import silence_loggers
 
 from lamin_cli._cache import cache
+from lamin_cli._io import io
 from lamin_cli._migration import migrate
 from lamin_cli._settings import settings
 
@@ -294,54 +301,60 @@ def info(schema: bool):
 @click.option("--name", type=str, default=None)
 @click.option("--uid", type=str, default=None)
 @click.option("--slug", type=str, default=None)
+@click.option("--permanent", is_flag=True, default=None, help="Permanently delete the entity where applicable, e.g., for artifact, transform, collection.")
 @click.option("--force", is_flag=True, default=False, help="Do not ask for confirmation (only relevant for instance).")
 # fmt: on
-def delete(entity: str, name: str | None = None, uid: str | None = None, slug: str | None = None, force: bool = False):
+def delete(entity: str, name: str | None = None, uid: str | None = None, slug: str | None = None, permanent: bool | None = None, force: bool = False):
     """Delete an entity.
 
     Currently supported: `branch`, `artifact`, `transform`, `collection`, and `instance`. For example:
 
     ```
     lamin delete https://lamin.ai/account/instance/artifact/e2G7k9EVul4JbfsEYAy5
+    lamin delete https://lamin.ai/account/instance/artifact/e2G7k9EVul4JbfsEYAy5 --permanent
     lamin delete branch --name my_branch
     lamin delete instance --slug account/name
     ```
     """
     from lamin_cli._delete import delete as delete_
 
-    return delete_(entity=entity, name=name, uid=uid, slug=slug, force=force)
+    return delete_(entity=entity, name=name, uid=uid, slug=slug, permanent=permanent, force=force)
 
 
 @main.command()
-@click.argument("entity", type=str)
+@click.argument("entity", type=str, required=False)
 @click.option("--uid", help="The uid for the entity.")
 @click.option("--key", help="The key for the entity.")
 @click.option(
     "--with-env", is_flag=True, help="Also return the environment for a tranform."
 )
-def load(entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False):
+def load(entity: str | None = None, uid: str | None = None, key: str | None = None, with_env: bool = False):
     """Load a file or folder into the cache or working directory.
 
-    Pass a URL, `artifact`, or `transform`. For example:
+    Pass a URL or `--key`. For example:
 
     ```
     lamin load https://lamin.ai/account/instance/artifact/e2G7k9EVul4JbfsE
-    lamin load artifact --key mydatasets/mytable.parquet
+    lamin load --key mydatasets/mytable.parquet
+    lamin load --key analysis.ipynb
+    lamin load --key myanalyses/analysis.ipynb --with-env
+    ```
+
+    You can also pass a uid and the entity type:
+
+    ```
     lamin load artifact --uid e2G7k9EVul4JbfsE
-    lamin load transform --key analysis.ipynb
     lamin load transform --uid Vul4JbfsEYAy5
-    lamin load transform --uid Vul4JbfsEYAy5 --with-env
     ```
     """
-    is_slug = entity.count("/") == 1
-    if is_slug:
-        from lamindb_setup._connect_instance import _connect_cli
-        # for backward compat and convenience, connect to the instance
-        return _connect_cli(entity)
-    else:
-        from lamin_cli._load import load as load_
-
-        return load_(entity, uid=uid, key=key, with_env=with_env)
+    from lamin_cli._load import load as load_
+    if entity is not None:
+        is_slug = entity.count("/") == 1
+        if is_slug:
+            from lamindb_setup._connect_instance import _connect_cli
+            # for backward compat
+            return _connect_cli(entity)
+    return load_(entity, uid=uid, key=key, with_env=with_env)
 
 
 def _describe(entity: str = "artifact", uid: str | None = None, key: str | None = None):
@@ -539,6 +552,7 @@ def run(filepath: str, project: str, image_url: str, packages: str, cpu: int, gp
 main.add_command(settings)
 main.add_command(cache)
 main.add_command(migrate)
+main.add_command(io)
 
 # https://stackoverflow.com/questions/57810659/automatically-generate-all-help-documentation-for-click-commands
 # https://claude.ai/chat/73c28487-bec3-4073-8110-50d1a2dd6b84

lamin_cli/_delete.py

@@ -9,6 +9,7 @@ def delete(
     name: str | None = None,
     uid: str | None = None,
     slug: str | None = None,
+    permanent: bool | None = None,
     force: bool = False,
 ):
     # TODO: refactor to abstract getting and deleting across entities
@@ -21,22 +22,22 @@ def delete(
         assert name is not None, "You have to pass a name for deleting a branch."
         from lamindb import Branch
 
-        Branch.get(name=name).delete()
+        Branch.get(name=name).delete(permanent=permanent)
     elif entity == "artifact":
         assert uid is not None, "You have to pass a uid for deleting an artifact."
         from lamindb import Artifact
 
-        Artifact.get(uid).delete()
+        Artifact.get(uid).delete(permanent=permanent)
     elif entity == "transform":
         assert uid is not None, "You have to pass a uid for deleting an transform."
         from lamindb import Transform
 
-        Transform.get(uid).delete()
+        Transform.get(uid).delete(permanent=permanent)
     elif entity == "collection":
         assert uid is not None, "You have to pass a uid for deleting an collection."
         from lamindb import Collection
 
-        Collection.get(uid).delete()
+        Collection.get(uid).delete(permanent=permanent)
     elif entity == "instance":
         return delete_instance(slug, force=force)
     else:  # backwards compatibility

lamin_cli/_io.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import lamindb_setup as ln_setup
+
+from lamin_cli.clone._clone_verification import (
+    _count_instance_records,
+)
+
+if os.environ.get("NO_RICH"):
+    import click as click
+else:
+    import rich_click as click
+
+
+@click.group()
+def io():
+    """Import and export instances."""
+
+
+# fmt: off
+@io.command("snapshot")
+@click.option("--upload/--no-upload", is_flag=True, help="Whether to upload the snapshot.", default=True)
+@click.option("--track/--no-track", is_flag=True, help="Whether to track snapshot generation.", default=True)
+# fmt: on
+def snapshot(upload: bool, track: bool) -> None:
+    """Create and optionally upload a SQLite snapshot of the current instance."""
+    if not ln_setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    instance_owner = ln_setup.settings.instance.owner
+    instance_name = ln_setup.settings.instance.name
+
+    ln_setup.connect(f"{instance_owner}/{instance_name}", use_root_db_user=True)
+
+    import lamindb as ln
+
+    original_counts = _count_instance_records()
+
+    modules_without_lamindb = ln_setup.settings.instance.modules
+    modules_complete = modules_without_lamindb.copy()
+    modules_complete.add("lamindb")
+
+
+    with tempfile.TemporaryDirectory() as export_dir:
+        if track:
+            ln.track()
+        ln_setup.io.export_db(module_names=modules_complete, output_dir=export_dir)
+        if track:
+            ln.finish()
+
+        script_path = (
+            Path(__file__).parent / "clone" / "create_sqlite_clone_and_import_db.py"
+        )
+        result = subprocess.run(
+            [
+                sys.executable,
+                str(script_path),
+                "--instance-name",
+                instance_name,
+                "--export-dir",
+                export_dir,
+                "--modules",
+                ",".join(modules_without_lamindb),
+                "--original-counts",
+                json.dumps(original_counts),
+            ],
+            check=False,
+            stderr=subprocess.PIPE,
+            text=True,
+            cwd=Path.cwd(),
+        )
+        if result.returncode != 0:
+            try:
+                mismatches = json.loads(result.stderr.strip())
+                error_msg = "Record count mismatch detected:\n" + "\n".join(
+                    [f"  {table}: original={orig}, clone={clone}"
+                    for table, (orig, clone) in mismatches.items()]
+                )
+                raise click.ClickException(error_msg)
+            except json.JSONDecodeError:
+                error_msg = f"Clone verification failed:\n{result.stderr}"
+
+
+        ln_setup.connect(f"{instance_owner}/{instance_name}", use_root_db_user=True)
+        if upload:
+            ln_setup.core._clone.upload_sqlite_clone(
+                local_sqlite_path=f"{instance_name}-clone/.lamindb/lamin.db",
+                compress=True,
+            )
+
+        ln_setup.disconnect()
+
+
+# fmt: off
+@io.command("exportdb")
+@click.option("--modules", type=str, default=None, help="Comma-separated list of modules to export (e.g., 'lamindb,bionty').",)
+@click.option("--output-dir", type=str, help="Output directory for exported parquet files.")
+@click.option("--max-workers", type=int, default=8, help="Number of parallel workers.")
+@click.option("--chunk-size", type=int, default=500_000, help="Number of rows per chunk for large tables.")
+# fmt: on
+def exportdb(modules: str | None, output_dir: str, max_workers: int, chunk_size: int):
+    """Export registry tables to parquet files."""
+    if not ln_setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    module_list = modules.split(",") if modules else None
+    ln_setup.io.export_db(
+        module_names=module_list,
+        output_dir=output_dir,
+        max_workers=max_workers,
+        chunk_size=chunk_size,
+    )
+
+
+# fmt: off
+@io.command("importdb")
+@click.option("--modules", type=str, default=None, help="Comma-separated list of modules to import (e.g., 'lamindb,bionty').")
+@click.option("--input-dir", type=str, help="Input directory containing exported parquet files.")
+@click.option("--if-exists", type=click.Choice(["fail", "replace", "append"]), default="replace", help="How to handle existing data.")
+# fmt: on
+def importdb(modules: str | None, input_dir: str, if_exists: str):
+    """Import registry tables from parquet files."""
+    if not ln_setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    module_list = modules.split(",") if modules else None
+    ln_setup.io.import_db(
+        module_names=module_list,
+        input_dir=input_dir,
+        if_exists=if_exists,
+    )

lamin_cli/_load.py

@@ -6,15 +6,35 @@ from pathlib import Path
 
 from lamin_utils import logger
 
-from ._save import parse_title_r_notebook
+from ._save import infer_registry_from_path, parse_title_r_notebook
 from .urls import decompose_url
 
 
 def load(
-    entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False
+    entity: str | None = None,
+    uid: str | None = None,
+    key: str | None = None,
+    with_env: bool = False,
 ):
+    """Load artifact, collection, or transform from LaminDB.
+
+    Args:
+        entity: URL containing 'lamin', or 'artifact', 'collection', or 'transform'
+        uid: Unique identifier (prefix matching supported)
+        key: Key identifier
+        with_env: If True, also load environment requirements file for transforms
+
+    Returns:
+        Path to loaded transform, or None for artifacts/collections
+    """
     import lamindb_setup as ln_setup
 
+    if entity is None:
+        if key is None:
+            raise SystemExit("Either entity or key has to be provided.")
+        else:
+            entity = infer_registry_from_path(key)
+
     if entity.startswith("https://") and "lamin" in entity:
         url = entity
         instance, entity, uid = decompose_url(url)
@@ -102,27 +122,25 @@ def load(
                 transforms = transforms.order_by("-created_at")
             transform = transforms.first()
 
-            target_relpath = Path(transform.key)
-            if len(target_relpath.parents) > 1:
-                logger.important(
-                    "preserve the folder structure for versioning:"
-                    f" {target_relpath.parent}/"
-                )
-                target_relpath.parent.mkdir(parents=True, exist_ok=True)
-            if target_relpath.exists():
-                response = input(f"! {target_relpath} exists: replace? (y/n)")
+            target_path = Path(transform.key)
+            if ln_setup.settings.dev_dir is not None:
+                target_path = ln_setup.settings.dev_dir / target_path
+            if len(target_path.parents) > 1:
+                target_path.parent.mkdir(parents=True, exist_ok=True)
+            if target_path.exists():
+                response = input(f"! {target_path} exists: replace? (y/n)")
                 if response != "y":
                     raise SystemExit("Aborted.")
 
             if transform.source_code is not None:
-                if target_relpath.suffix in (".ipynb", ".Rmd", ".qmd"):
-                    script_to_notebook(transform, target_relpath, bump_revision=True)
+                if target_path.suffix in (".ipynb", ".Rmd", ".qmd"):
+                    script_to_notebook(transform, target_path, bump_revision=True)
                 else:
-                    target_relpath.write_text(transform.source_code)
+                    target_path.write_text(transform.source_code)
             else:
                 raise SystemExit("No source code available for this transform.")
 
-            logger.important(f"{transform.type} is here: {target_relpath}")
+            logger.important(f"{transform.type} is here: {target_path}")
 
             if with_env:
                 ln.settings.track_run_inputs = False
@@ -132,8 +150,7 @@ def load(
                 ):
                     filepath_env_cache = transform.latest_run.environment.cache()
                     target_env_filename = (
-                        target_relpath.parent
-                        / f"{target_relpath.stem}__requirements.txt"
+                        target_path.parent / f"{target_path.stem}__requirements.txt"
                     )
                     shutil.move(filepath_env_cache, target_env_filename)
                     logger.important(f"environment is here: {target_env_filename}")
@@ -142,14 +159,13 @@ def load(
                         "latest transform run with environment doesn't exist"
                     )
 
-            return target_relpath
+            return target_path
         case "artifact" | "collection":
             ln.settings.track_run_inputs = False
 
             EntityClass = ln.Artifact if entity == "artifact" else ln.Collection
 
-            # we don't use .get here because DoesNotExist is hard to catch
-            # due to private django API
+            # we don't use .get here because DoesNotExist is hard to catch due to private django API
             # we use `.objects` here because we don't want to exclude kind = __lamindb_run__ artifacts
             if query_by_uid:
                 entities = EntityClass.objects.filter(uid__startswith=uid)

lamin_cli/_migration.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import os
-from typing import Optional
 
 if os.environ.get("NO_RICH"):
     import click as click

lamin_cli/_save.py

@@ -2,11 +2,10 @@ from __future__ import annotations
 
 import os
 import re
-import sys
 from pathlib import Path
-from typing import TYPE_CHECKING
 
 import click
+import lamindb_setup as ln_setup
 from lamin_utils import logger
 from lamindb_setup.core.hashing import hash_file
 
@@ -86,15 +85,15 @@ def save(
     from lamindb_setup.core.upath import LocalPathClasses, UPath, create_path
 
     # this allows to have the correct treatment of credentials in case of cloud paths
-    path = create_path(path)
+    ppath = create_path(path)
     # isinstance is needed to cast the type of path to UPath
     # to avoid mypy erors
-    assert isinstance(path, UPath)
-    if not path.exists():
-        raise click.BadParameter(f"Path {path} does not exist", param_hint="path")
+    assert isinstance(ppath, UPath)
+    if not ppath.exists():
+        raise click.BadParameter(f"Path {ppath} does not exist", param_hint="path")
 
     if registry is None:
-        registry = infer_registry_from_path(path)
+        registry = infer_registry_from_path(ppath)
 
     if project is not None:
         project_record = ln.Project.filter(
@@ -121,7 +120,7 @@ def save(
                 f"Branch '{branch}' not found, either create it with `ln.Branch(name='...').save()` or fix typos."
             )
 
-    is_cloud_path = not isinstance(path, LocalPathClasses)
+    is_cloud_path = not isinstance(ppath, LocalPathClasses)
 
     if registry == "artifact":
         ln.settings.creation.artifact_silence_missing_run_warning = True
@@ -144,7 +143,7 @@ def save(
             return "missing-key-or-description"
 
         artifact = ln.Artifact(
-            path,
+            ppath,
             key=key,
             description=description,
             revises=revises,
@@ -167,34 +166,40 @@ def save(
     if registry == "transform":
         if key is not None:
             logger.warning(
-                "key is ignored for transforms, the transform key is determined by the filename"
+                "key is ignored for transforms, the transform key is determined by the filename and the development directory (dev-dir)"
             )
         if is_cloud_path:
             logger.error("Can not register a transform from a cloud path")
             return "transform-with-cloud-path"
 
-        if path.suffix in {".qmd", ".Rmd"}:
-            html_file_exists = path.with_suffix(".html").exists()
-            nb_html_file_exists = path.with_suffix(".nb.html").exists()
+        if ppath.suffix in {".qmd", ".Rmd"}:
+            html_file_exists = ppath.with_suffix(".html").exists()
+            nb_html_file_exists = ppath.with_suffix(".nb.html").exists()
 
             if not html_file_exists and not nb_html_file_exists:
                 logger.error(
-                    f"Please export your {path.suffix} file as an html file here"
-                    f" {path.with_suffix('.html')}"
+                    f"Please export your {ppath.suffix} file as an html file here"
+                    f" {ppath.with_suffix('.html')}"
                 )
                 return "export-qmd-Rmd-as-html"
             elif html_file_exists and nb_html_file_exists:
                 logger.error(
-                    f"Please delete one of\n - {path.with_suffix('.html')}\n -"
-                    f" {path.with_suffix('.nb.html')}"
+                    f"Please delete one of\n - {ppath.with_suffix('.html')}\n -"
+                    f" {ppath.with_suffix('.nb.html')}"
                 )
                 return "delete-html-or-nb-html"
 
-        content = path.read_text()
-        uid = parse_uid_from_code(content, path.suffix)
+        content = ppath.read_text()
+        uid = parse_uid_from_code(content, ppath.suffix)
+
+        ppath = ppath.resolve().expanduser()
+        if ln_setup.settings.dev_dir is not None:
+            key = ppath.relative_to(ln_setup.settings.dev_dir).as_posix()
+        else:
+            key = ppath.name
 
         if uid is not None:
-            logger.important(f"mapped '{path.name}' on uid '{uid}'")
+            logger.important(f"mapped '{ppath.name}' on uid '{uid}'")
             if len(uid) == 16:
                 # is full uid
                 transform = ln.Transform.filter(uid=uid).one_or_none()
@@ -214,12 +219,16 @@ def save(
                 if transform is None:
                     uid = f"{stem_uid}0000"
         else:
-            # TODO: account for folders as we do in ln.track()
-            transform_hash, _ = hash_file(path)
-            transform = ln.Transform.filter(key=path.name, is_latest=True).one_or_none()
+            transform_hash, _ = hash_file(ppath)
+            transform = ln.Transform.filter(hash=transform_hash).first()
             if transform is not None and transform.hash is not None:
                 if transform.hash == transform_hash:
                     if transform.type != "notebook":
+                        logger.important(f"transform already saved: {transform}")
+                        if transform.key != key:
+                            transform.key = key
+                            logger.important(f"updated key to '{key}'")
+                            transform.save()
                         return None
                     if os.getenv("LAMIN_TESTING") == "true":
                         response = "y"
@@ -245,21 +254,21 @@ def save(
             if revises is None:
                 raise ln.errors.InvalidArgument("The stem uid is not found.")
         if transform is None:
-            if path.suffix == ".ipynb":
+            if ppath.suffix == ".ipynb":
                 from nbproject.dev import read_notebook
                 from nbproject.dev._meta_live import get_title
 
-                nb = read_notebook(path)
+                nb = read_notebook(ppath)
                 description = get_title(nb)
-            elif path.suffix in {".qmd", ".Rmd"}:
+            elif ppath.suffix in {".qmd", ".Rmd"}:
                 description = parse_title_r_notebook(content)
             else:
                 description = None
             transform = ln.Transform(
                 uid=uid,
                 description=description,
-                key=path.name,
-                type="script" if path.suffix in {".R", ".py"} else "notebook",
+                key=key,
+                type="script" if ppath.suffix in {".R", ".py"} else "notebook",
                 revises=revises,
             )
             if space is not None:
@@ -267,7 +276,9 @@ def save(
             if branch is not None:
                 transform.branch = branch_record
             transform.save()
-            logger.important(f"created Transform('{transform.uid}')")
+            logger.important(
+                f"created Transform('{transform.uid}', key='{transform.key}')"
+            )
         if project is not None:
             transform.projects.add(project_record)
             logger.important(f"labeled with project: {project_record.name}")
@@ -292,7 +303,7 @@ def save(
         return_code = save_context_core(
             run=run,
             transform=transform,
-            filepath=path,
+            filepath=ppath,
             from_cli=True,
         )
         return return_code

lamin_cli/_settings.py

@@ -22,7 +22,7 @@ def settings(ctx):
 
     Allows to get and set these settings:
 
-    - `work-dir` → {attr}`~lamindb.setup.core.SetupSettings.work_dir`
+    - `dev-dir` → {attr}`~lamindb.setup.core.SetupSettings.dev_dir`
     - `private-django-api` → {attr}`~lamindb.setup.core.SetupSettings.private_django_api`
     - `branch` → current branch (use `lamin switch --branch` to change)
     - `space` → current space (use `lamin switch --space` to change)
@@ -30,16 +30,16 @@ def settings(ctx):
     Examples for getting a setting:
 
     ```
-    lamin settings get work-dir
+    lamin settings get dev-dir
     lamin settings get branch
     ```
 
     Examples for setting the working directory:
 
     ```
-    lamin settings set work-dir .  # set work-dir to current directory
-    lamin settings set work-dir ~/my-project  # set work-dir to ~/my-project
-    lamin settings set work-dir none  # unset work-dir
+    lamin settings set dev-dir .  # set dev-dir to current directory
+    lamin settings set dev-dir ~/my-project  # set dev-dir to ~/my-project
+    lamin settings set dev-dir none  # unset dev-dir
     ```
     """
     if ctx.invoked_subcommand is None:
@@ -53,7 +53,7 @@ def settings(ctx):
 @click.argument(
     "setting",
     type=click.Choice(
-        ["auto-connect", "private-django-api", "work-dir"], case_sensitive=False
+        ["auto-connect", "private-django-api", "dev-dir"], case_sensitive=False
     ),
 )
 @click.argument("value")  # No explicit type - let Click handle it
@@ -65,17 +65,17 @@ def set(setting: str, value: str):
         settings_.auto_connect = click.BOOL(value)
     if setting == "private-django-api":
         settings_.private_django_api = click.BOOL(value)
-    if setting == "work-dir":
+    if setting == "dev-dir":
         if value.lower() == "none":
             value = None  # type: ignore[assignment]
-        settings_.work_dir = value
+        settings_.dev_dir = value
 
 
 @settings.command("get")
 @click.argument(
     "setting",
     type=click.Choice(
-        ["auto-connect", "private-django-api", "space", "branch", "work-dir"],
+        ["auto-connect", "private-django-api", "space", "branch", "dev-dir"],
         case_sensitive=False,
     ),
 )
@@ -87,8 +87,8 @@ def get(setting: str):
         _, value = settings_._read_branch_idlike_name()
     elif setting == "space":
         _, value = settings_._read_space_idlike_name()
-    elif setting == "work-dir":
-        value = settings_.work_dir
+    elif setting == "dev-dir":
+        value = settings_.dev_dir
         if value is None:
             value = "None"
     else:

lamin_cli/clone/_clone_verification.py

@@ -0,0 +1,56 @@
+from concurrent.futures import ThreadPoolExecutor
+
+from django.db import OperationalError, ProgrammingError
+from lamin_utils import logger
+
+
+def _count_instance_records() -> dict[str, int]:
+    """Count all records across SQLRecord registries in parallel.
+
+    Returns:
+        Dictionary mapping table names (format: "app_label.ModelName") to  their record counts.
+
+    Example:
+        >>> counts = _count_all_records()
+        >>> counts
+        {'lamindb.Artifact': 1523, 'lamindb.Collection': 42, 'bionty.Gene': 60000}
+    """
+    # Import here to ensure that models are loaded
+    from django.apps import apps
+    from lamindb.models import SQLRecord
+
+    def _count_model(model):
+        """Count records for a single model."""
+        table_name = f"{model._meta.app_label}.{model.__name__}"
+        try:
+            return (table_name, model.objects.count())
+        except (OperationalError, ProgrammingError) as e:
+            logger.warning(f"Could not count {table_name}: {e}")
+            return (table_name, 0)
+
+    models = [m for m in apps.get_models() if issubclass(m, SQLRecord)]
+
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        results = executor.map(_count_model, models)
+
+    return dict(results)
+
+
+def _compare_record_counts(
+    original: dict[str, int], clone: dict[str, int]
+) -> dict[str, tuple[int, int]]:
+    """Compare record counts and return mismatches."""
+    mismatches = {}
+
+    all_tables = set(original.keys()) | set(clone.keys())
+
+    for table in all_tables:
+        orig_count = original.get(table, 0)
+        clone_count = clone.get(table, 0)
+
+        # we allow a difference of 1 because of tracking
+        # new records during the cloning process
+        if abs(clone_count - orig_count) > 1:
+            mismatches[table] = (orig_count, clone_count)
+
+    return mismatches

lamin_cli/clone/create_sqlite_clone_and_import_db.py

@@ -0,0 +1,51 @@
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import lamindb_setup as ln_setup
+
+if __name__ == "__main__":
+    sys.path.insert(0, str(Path(__file__).parent.parent))
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--instance-name", required=True)
+    parser.add_argument("--export-dir", required=True)
+    parser.add_argument("--modules", required=True)
+    parser.add_argument("--original-counts", required=True)
+    args = parser.parse_args()
+
+    instance_name = args.instance_name
+    export_dir = args.export_dir
+    modules_without_lamindb = {m for m in args.modules.split(",") if m}
+    modules_complete = modules_without_lamindb.copy()
+    modules_complete.add("lamindb")
+
+    ln_setup.init(
+        storage=f"{instance_name}-clone", modules=f"{','.join(modules_without_lamindb)}"
+    )
+
+    ln_setup.io.import_db(
+        module_names=list(modules_complete), input_dir=export_dir, if_exists="replace"
+    )
+
+    from django.db import connection
+
+    with connection.cursor() as cursor:
+        cursor.execute("PRAGMA wal_checkpoint(FULL)")
+
+    from lamin_cli.clone._clone_verification import (
+        _compare_record_counts,
+        _count_instance_records,
+    )
+
+    clone_counts = _count_instance_records()
+    original_counts = json.loads(args.original_counts)
+    mismatches = _compare_record_counts(original_counts, clone_counts)
+    if mismatches:
+        print(json.dumps(mismatches), file=sys.stderr)
+        sys.exit(1)
+
+    from django.db import connections
+
+    connections.close_all()

{lamin_cli-1.8.2.dist-info → lamin_cli-1.10.0.dist-info}/METADATA

@@ -1,9 +1,10 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: lamin_cli
-Version: 1.8.2
+Version: 1.10.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: rich-click>=1.7
 Project-URL: Home, https://github.com/laminlabs/lamin-cli
 

lamin_cli-1.10.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+lamin_cli/__init__.py,sha256=EEqwDNhBI_f1OthJACbb7XrDCxzXxoPuIkCP0wbdWdk,605
+lamin_cli/__main__.py,sha256=zGMlXpDgKZpfQqIsedPnk0S8-jy0NEFcF4amtyIOHE0,19721
+lamin_cli/_annotate.py,sha256=ZD76__K-mQt7UpYqyM1I2lKCs-DraTmnkjsByIHmD-g,1839
+lamin_cli/_cache.py,sha256=oplwE8AcS_9PYptQUZxff2qTIdNFS81clGPkJNWk098,800
+lamin_cli/_delete.py,sha256=1wp8LQdiWHJznRrm4abxEhApB9derBWEm25ufrzjvIg,1531
+lamin_cli/_io.py,sha256=v103aEmMPegQQJGUfsrAB5rzdLN4NqkN07ZxhglTv7o,4989
+lamin_cli/_load.py,sha256=OqkOJe2Afq4ZrN5abM1ueqaDDvm1psdnQmzhXTTWe1I,8233
+lamin_cli/_migration.py,sha256=w-TLYC2q6sqcfNtIUewqRW_69_xy_71X3eSV7Ud58jk,1239
+lamin_cli/_save.py,sha256=bysKSkivcvELW1srOoc772udOzREEnFRVPUClBJ9Qac,12084
+lamin_cli/_settings.py,sha256=nqVM8FO9kjL0SXYqP8x-Xfww6Qth4I86HKKeTirsgc4,2657
+lamin_cli/urls.py,sha256=gc72s4SpaAQA8J50CtCIWlr49DWOSL_a6OM9lXfPouM,367
+lamin_cli/clone/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lamin_cli/clone/_clone_verification.py,sha256=8hRFZt_1Z0i2wMqKBkrioq01RJEjwHy9cYmw6LV4PXI,1835
+lamin_cli/clone/create_sqlite_clone_and_import_db.py,sha256=yv2mLkHdRUao_IkZO_4Swu5w4Nieh1gMrBwPbyn4c1o,1544
+lamin_cli/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lamin_cli/compute/modal.py,sha256=QnR7GyyvWWWkLnou95HxS9xxSQfw1k-SiefM_qRVnU0,6010
+lamin_cli-1.10.0.dist-info/entry_points.txt,sha256=Qms85i9cZPlu-U7RnVZhFsF7vJ9gaLZUFkCjcGcXTpg,49
+lamin_cli-1.10.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamin_cli-1.10.0.dist-info/WHEEL,sha256=Dyt6SBfaasWElUrURkknVFAZDHSTwxg3PaTza7RSbkY,100
+lamin_cli-1.10.0.dist-info/METADATA,sha256=QIppUnCd0P8oMGAcvCSREQo5De9ifM_uomSTgfWdH6k,360
+lamin_cli-1.10.0.dist-info/RECORD,,

{lamin_cli-1.8.2.dist-info → lamin_cli-1.10.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: flit 3.10.1
+Generator: flit 3.12.0
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

lamin_cli-1.8.2.dist-info/RECORD

@@ -1,17 +0,0 @@
-lamin_cli/__init__.py,sha256=UE_o2BSyhfXL0PkGbgUKzFvRh9nw9oZf7sdX1gNf0G0,563
-lamin_cli/__main__.py,sha256=zuzvpfWDvrLxvcLNwAyGQHIGXiFvAra4dnPe7Nygu6M,19225
-lamin_cli/_annotate.py,sha256=ZD76__K-mQt7UpYqyM1I2lKCs-DraTmnkjsByIHmD-g,1839
-lamin_cli/_cache.py,sha256=oplwE8AcS_9PYptQUZxff2qTIdNFS81clGPkJNWk098,800
-lamin_cli/_delete.py,sha256=dxItUgf7At247iYGLagmJ2Ccp5nAWgodPL7c7zfZQ_0,1420
-lamin_cli/_load.py,sha256=OgTGqZQtoJ0GYOhuJihz-izt0F4jM4U_nSX_vhPoukg,7698
-lamin_cli/_migration.py,sha256=XUl_L9_3pTTk5jJoBiqbzf0Bd2LdKKtHa1zPZ4Rla5c,1267
-lamin_cli/_save.py,sha256=EPYsRpl0vD_Q52ksqVyP56ZOvPXBjgTFPeJWyGokN8I,11555
-lamin_cli/_settings.py,sha256=mTnwU5zfBGwt9x6RrhFpi4f0DeyEfI02YWs96x_JYWw,2672
-lamin_cli/urls.py,sha256=gc72s4SpaAQA8J50CtCIWlr49DWOSL_a6OM9lXfPouM,367
-lamin_cli/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lamin_cli/compute/modal.py,sha256=QnR7GyyvWWWkLnou95HxS9xxSQfw1k-SiefM_qRVnU0,6010
-lamin_cli-1.8.2.dist-info/entry_points.txt,sha256=Qms85i9cZPlu-U7RnVZhFsF7vJ9gaLZUFkCjcGcXTpg,49
-lamin_cli-1.8.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamin_cli-1.8.2.dist-info/WHEEL,sha256=ssQ84EZ5gH1pCOujd3iW7HClo_O_aDaClUbX4B8bjKY,100
-lamin_cli-1.8.2.dist-info/METADATA,sha256=ISh-SL8AGEvoOOSIkaUREwzYdLVk4U01NVAgySnzA20,337
-lamin_cli-1.8.2.dist-info/RECORD,,