lamin_cli 1.1.0__tar.gz → 1.2.0__tar.gz

{lamin_cli-1.1.0 → lamin_cli-1.2.0}/.pre-commit-config.yaml

@@ -16,8 +16,8 @@ repos:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --unsafe-fixes]
       - id: ruff-format
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v4.0.0-alpha.8
+  - repo: https://github.com/rbubley/mirrors-prettier
+    rev: v3.5.1
     hooks:
       - id: prettier
   - repo: https://github.com/kynan/nbstripout

{lamin_cli-1.1.0 → lamin_cli-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamin_cli
-Version: 1.1.0
+Version: 1.2.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

lamin_cli-1.2.0/lamin_cli/__init__.py

@@ -0,0 +1,3 @@
+"""Lamin CLI."""
+
+__version__ = "1.2.0"

{lamin_cli-1.1.0 → lamin_cli-1.2.0}/lamin_cli/__main__.py

@@ -279,18 +279,31 @@ def load(entity: str, uid: str | None = None, key: str | None = None, with_env:
 @click.argument("entity", type=str)
 @click.option("--uid", help="The uid for the entity.")
 @click.option("--key", help="The key for the entity.")
-@click.option(
-    "--with-env", is_flag=True, help="Also return the environment for a tranform."
-)
-def get(entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False):
+def get(entity: str, uid: str | None = None, key: str | None = None):
     """Query metadata about an entity.
 
-    Currently only works for artifact & transform and behaves like `lamin load`.
+    Currently only works for artifact.
     """
-    from lamin_cli._load import load as load_
+    import lamindb_setup as ln_setup
+
+    from ._load import decompose_url
 
-    click.echo(f"! to load a file or folder, please use: lamin load {entity}")
-    return load_(entity, uid=uid, key=key, with_env=with_env)
+    if entity.startswith("https://") and "lamin" in entity:
+        url = entity
+        instance, entity, uid = decompose_url(url)
+    elif entity not in {"artifact"}:
+        raise SystemExit("Entity has to be a laminhub URL or 'artifact'")
+    else:
+        instance = ln_setup.settings.instance.slug
+
+    ln_setup.connect(instance)
+    import lamindb as ln
+
+    if uid is not None:
+        artifact = ln.Artifact.get(uid)
+    else:
+        artifact = ln.Artifact.get(key=key)
+    artifact.describe()
 
 
 @main.command()

lamin_cli-1.2.0/lamin_cli/_load.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import re
+import shutil
+from pathlib import Path
+
+from lamin_utils import logger
+
+
+def decompose_url(url: str) -> tuple[str, str, str]:
+    assert any(keyword in url for keyword in ["transform", "artifact", "collection"])
+    for entity in ["transform", "artifact", "collection"]:
+        if entity in url:
+            break
+    uid = url.split(f"{entity}/")[1]
+    instance_slug = "/".join(url.split("/")[3:5])
+    return instance_slug, entity, uid
+
+
+def load(
+    entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False
+):
+    import lamindb_setup as ln_setup
+
+    if entity.startswith("https://") and "lamin" in entity:
+        url = entity
+        instance, entity, uid = decompose_url(url)
+    elif entity not in {"artifact", "transform", "collection"}:
+        raise SystemExit(
+            "Entity has to be a laminhub URL or 'artifact', 'collection', or 'transform'"
+        )
+    else:
+        instance = ln_setup.settings.instance.slug
+
+    ln_setup.connect(instance)
+    import lamindb as ln
+
+    def script_to_notebook(
+        transform: ln.Transform, notebook_path: Path, bump_revision: bool = False
+    ) -> None:
+        import jupytext
+        from lamin_utils._base62 import increment_base62
+
+        if notebook_path.suffix == ".ipynb":
+            # below is backward compat
+            if "# # transform.name" in transform.source_code:
+                new_content = transform.source_code.replace(
+                    "# # transform.name", f"# # {transform.description}"
+                )
+            elif transform.source_code.startswith("# %% [markdown]"):
+                source_code_split = transform.source_code.split("\n")
+                if source_code_split[1] == "#":
+                    source_code_split[1] = f"# # {transform.description}"
+                new_content = "\n".join(source_code_split)
+            else:
+                new_content = transform.source_code
+        else:  # R notebook
+            # Pattern to match title only within YAML header section
+            title_pattern = r'^---\n.*?title:\s*"([^"]*)".*?---'
+            title_match = re.search(
+                title_pattern, transform.source_code, flags=re.DOTALL | re.MULTILINE
+            )
+            new_content = transform.source_code
+            if title_match:
+                current_title = title_match.group(1)
+                if current_title != transform.description:
+                    pattern = r'^(---\n.*?title:\s*)"([^"]*)"(.*?---)'
+                    replacement = f'\\1"{transform.description}"\\3'
+                    new_content = re.sub(
+                        pattern,
+                        replacement,
+                        new_content,
+                        flags=re.DOTALL | re.MULTILINE,
+                    )
+                    logger.important(
+                        f"updated title to match description: {current_title} →"
+                        f" {transform.description}"
+                    )
+        if bump_revision:
+            uid = transform.uid
+            if uid in new_content:
+                new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
+                new_content = new_content.replace(uid, new_uid)
+                logger.important(f"updated uid: {uid} → {new_uid}")
+        if notebook_path.suffix == ".ipynb":
+            notebook = jupytext.reads(new_content, fmt="py:percent")
+            jupytext.write(notebook, notebook_path)
+        else:
+            notebook_path.write_text(new_content)
+
+    query_by_uid = uid is not None
+
+    match entity:
+        case "transform":
+            if query_by_uid:
+                # we don't use .get here because DoesNotExist is hard to catch
+                # due to private django API
+                # here full uid is not expected anymore as before
+                # via ln.Transform.objects.get(uid=uid)
+                transforms = ln.Transform.objects.filter(uid__startswith=uid)
+            else:
+                # if below, we take is_latest=True as the criterion, we might get draft notebooks
+                # hence, we use source_code__isnull=False and order by created_at instead
+                transforms = ln.Transform.objects.filter(
+                    key=key, source_code__isnull=False
+                )
+
+            if (n_transforms := len(transforms)) == 0:
+                err_msg = f"uid {uid}" if query_by_uid else f"key={key} and source_code"
+                raise SystemExit(f"Transform with {err_msg} does not exist.")
+
+            if n_transforms > 1:
+                transforms = transforms.order_by("-created_at")
+            transform = transforms.first()
+
+            target_relpath = Path(transform.key)
+            if len(target_relpath.parents) > 1:
+                logger.important(
+                    "preserve the folder structure for versioning:"
+                    f" {target_relpath.parent}/"
+                )
+                target_relpath.parent.mkdir(parents=True, exist_ok=True)
+            if target_relpath.exists():
+                response = input(f"! {target_relpath} exists: replace? (y/n)")
+                if response != "y":
+                    raise SystemExit("Aborted.")
+
+            if transform.source_code is not None:
+                if target_relpath.suffix in (".ipynb", ".Rmd", ".qmd"):
+                    script_to_notebook(transform, target_relpath, bump_revision=True)
+                else:
+                    target_relpath.write_text(transform.source_code)
+            else:
+                raise SystemExit("No source code available for this transform.")
+
+            logger.important(f"{transform.type} is here: {target_relpath}")
+
+            if with_env:
+                ln.settings.track_run_inputs = False
+                if (
+                    transform.latest_run is not None
+                    and transform.latest_run.environment is not None
+                ):
+                    filepath_env_cache = transform.latest_run.environment.cache()
+                    target_env_filename = (
+                        target_relpath.parent
+                        / f"{target_relpath.stem}__requirements.txt"
+                    )
+                    shutil.move(filepath_env_cache, target_env_filename)
+                    logger.important(f"environment is here: {target_env_filename}")
+                else:
+                    logger.warning(
+                        "latest transform run with environment doesn't exist"
+                    )
+
+            return target_relpath
+        case "artifact" | "collection":
+            ln.settings.track_run_inputs = False
+
+            EntityClass = ln.Artifact if entity == "artifact" else ln.Collection
+
+            # we don't use .get here because DoesNotExist is hard to catch
+            # due to private django API
+            if query_by_uid:
+                entities = EntityClass.filter(uid__startswith=uid)
+            else:
+                entities = EntityClass.filter(key=key)
+
+            if (n_entities := len(entities)) == 0:
+                err_msg = f"uid={uid}" if query_by_uid else f"key={key}"
+                raise SystemExit(
+                    f"{entity.capitalize()} with {err_msg} does not exist."
+                )
+
+            if n_entities > 1:
+                entities = entities.order_by("-created_at")
+
+            entity_obj = entities.first()
+            cache_path = entity_obj.cache()
+
+            # collection gives us a list of paths
+            if isinstance(cache_path, list):
+                logger.important(f"{entity} paths ({len(cache_path)} files):")
+                for i, path in enumerate(cache_path):
+                    if i < 5 or i >= len(cache_path) - 5:
+                        logger.important(f"  [{i + 1}/{len(cache_path)}] {path}")
+                    elif i == 5:
+                        logger.important(f"  ... {len(cache_path) - 10} more files ...")
+            else:
+                logger.important(f"{entity} is here: {cache_path}")
+        case _:
+            raise AssertionError(f"unknown entity {entity}")

{lamin_cli-1.1.0 → lamin_cli-1.2.0}/tests/test_load.py

@@ -60,7 +60,15 @@ def test_load_transform():
     path2.unlink()
 
 
-def test_load_artifact():
+def test_get_load_artifact():
+    result = subprocess.run(
+        "lamin get"
+        " 'https://lamin.ai/laminlabs/lamin-site-assets/artifact/e2G7k9EVul4JbfsEYAy5'",
+        shell=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0
+
     result = subprocess.run(
         "lamin load"
         " 'https://lamin.ai/laminlabs/lamin-site-assets/artifact/e2G7k9EVul4JbfsEYAy5'",
@@ -76,3 +84,12 @@ def test_load_artifact():
         capture_output=True,
     )
     assert result.returncode == 0
+
+
+def test_load_collection():
+    result = subprocess.run(
+        "lamin load 'https://lamin.ai/laminlabs/lamindata/collection/2wUs6V1OuGzp5Ll4'",
+        shell=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0

{lamin_cli-1.1.0 → lamin_cli-1.2.0}/tests/test_save_files.py

@@ -44,7 +44,7 @@ def test_save_file():
     )
     print(result.stdout.decode())
     print(result.stderr.decode())
-    assert "found artifact with same hash" in result.stdout.decode()
+    assert "returning existing artifact with same hash" in result.stdout.decode()
     assert "key='mytest'" in result.stdout.decode()
     assert "storage path:" in result.stdout.decode()
     assert result.returncode == 0

lamin_cli-1.1.0/lamin_cli/__init__.py

@@ -1,3 +0,0 @@
-"""Lamin CLI."""
-
-__version__ = "1.1.0"

lamin_cli-1.1.0/lamin_cli/_load.py

@@ -1,169 +0,0 @@
-from __future__ import annotations
-
-import re
-import shutil
-from pathlib import Path
-
-from lamin_utils import logger
-
-
-def decompose_url(url: str) -> tuple[str, str, str]:
-    assert "transform" in url or "artifact" in url
-    for entity in ["transform", "artifact"]:
-        if entity in url:
-            break
-    uid = url.split(f"{entity}/")[1]
-    instance_slug = "/".join(url.split("/")[3:5])
-    return instance_slug, entity, uid
-
-
-def load(
-    entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False
-):
-    import lamindb_setup as ln_setup
-
-    if entity.startswith("https://") and "lamin" in entity:
-        url = entity
-        instance, entity, uid = decompose_url(url)
-    elif entity not in {"artifact", "transform"}:
-        raise SystemExit("Entity has to be a laminhub URL or 'artifact' or 'transform'")
-    else:
-        instance = ln_setup.settings.instance.slug
-
-    ln_setup.connect(instance)
-    import lamindb as ln
-
-    def script_to_notebook(
-        transform: ln.Transform, notebook_path: Path, bump_revision: bool = False
-    ) -> None:
-        import jupytext
-        from lamin_utils._base62 import increment_base62
-
-        if notebook_path.suffix == ".ipynb":
-            # below is backward compat
-            if "# # transform.name" in transform.source_code:
-                new_content = transform.source_code.replace(
-                    "# # transform.name", f"# # {transform.description}"
-                )
-            elif transform.source_code.startswith("# %% [markdown]"):
-                source_code_split = transform.source_code.split("\n")
-                if source_code_split[1] == "#":
-                    source_code_split[1] = f"# # {transform.description}"
-                new_content = "\n".join(source_code_split)
-            else:
-                new_content = transform.source_code
-        else:  # R notebook
-            # Pattern to match title only within YAML header section
-            title_pattern = r'^---\n.*?title:\s*"([^"]*)".*?---'
-            title_match = re.search(
-                title_pattern, transform.source_code, flags=re.DOTALL | re.MULTILINE
-            )
-            new_content = transform.source_code
-            if title_match:
-                current_title = title_match.group(1)
-                if current_title != transform.description:
-                    pattern = r'^(---\n.*?title:\s*)"([^"]*)"(.*?---)'
-                    replacement = f'\\1"{transform.description}"\\3'
-                    new_content = re.sub(
-                        pattern,
-                        replacement,
-                        new_content,
-                        flags=re.DOTALL | re.MULTILINE,
-                    )
-                    logger.important(
-                        f"updated title to match description: {current_title} →"
-                        f" {transform.description}"
-                    )
-        if bump_revision:
-            uid = transform.uid
-            if uid in new_content:
-                new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
-                new_content = new_content.replace(uid, new_uid)
-                logger.important(f"updated uid: {uid} → {new_uid}")
-        if notebook_path.suffix == ".ipynb":
-            notebook = jupytext.reads(new_content, fmt="py:percent")
-            jupytext.write(notebook, notebook_path)
-        else:
-            notebook_path.write_text(new_content)
-
-    query_by_uid = uid is not None
-
-    if entity == "transform":
-        if query_by_uid:
-            # we don't use .get here because DoesNotExist is hard to catch
-            # due to private django API
-            # here full uid is not expected anymore as before
-            # via ln.Transform.objects.get(uid=uid)
-            transforms = ln.Transform.objects.filter(uid__startswith=uid)
-        else:
-            # if below, we take is_latest=True as the criterion, we might get draft notebooks
-            # hence, we use source_code__isnull=False and order by created_at instead
-            transforms = ln.Transform.objects.filter(key=key, source_code__isnull=False)
-
-        if (n_transforms := len(transforms)) == 0:
-            err_msg = f"uid {uid}" if query_by_uid else f"key={key} and source_code"
-            raise SystemExit(f"Transform with {err_msg} does not exist.")
-
-        if n_transforms > 1:
-            transforms = transforms.order_by("-created_at")
-        transform = transforms.first()
-
-        target_relpath = Path(transform.key)
-        if len(target_relpath.parents) > 1:
-            logger.important(
-                "preserve the folder structure for versioning:"
-                f" {target_relpath.parent}/"
-            )
-            target_relpath.parent.mkdir(parents=True, exist_ok=True)
-        if target_relpath.exists():
-            response = input(f"! {target_relpath} exists: replace? (y/n)")
-            if response != "y":
-                raise SystemExit("Aborted.")
-
-        if transform.source_code is not None:
-            if target_relpath.suffix in (".ipynb", ".Rmd", ".qmd"):
-                script_to_notebook(transform, target_relpath, bump_revision=True)
-            else:
-                target_relpath.write_text(transform.source_code)
-        else:
-            raise SystemExit("No source code available for this transform.")
-
-        logger.important(f"{transform.type} is here: {target_relpath}")
-
-        if with_env:
-            ln.settings.track_run_inputs = False
-            if (
-                transform.latest_run is not None
-                and transform.latest_run.environment is not None
-            ):
-                filepath_env_cache = transform.latest_run.environment.cache()
-                target_env_filename = (
-                    target_relpath.parent / f"{target_relpath.stem}__requirements.txt"
-                )
-                shutil.move(filepath_env_cache, target_env_filename)
-                logger.important(f"environment is here: {target_env_filename}")
-            else:
-                logger.warning("latest transform run with environment doesn't exist")
-
-        return target_relpath
-    elif entity == "artifact":
-        ln.settings.track_run_inputs = False
-
-        if query_by_uid:
-            # we don't use .get here because DoesNotExist is hard to catch
-            # due to private django API
-            artifacts = ln.Artifact.filter(uid__startswith=uid)
-        else:
-            artifacts = ln.Artifact.filter(key=key)
-
-        if (n_artifacts := len(artifacts)) == 0:
-            err_msg = f"uid={uid}" if query_by_uid else f"key={key}"
-            raise SystemExit(f"Artifact with {err_msg} does not exist.")
-
-        if n_artifacts > 1:
-            artifacts = artifacts.order_by("-created_at")
-        artifact = artifacts.first()
-
-        cache_path = artifact.cache()
-        logger.important(f"artifact is here: {cache_path}")
-        return cache_path