lamin_cli 1.0.7__tar.gz → 1.2.0__tar.gz

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/.gitignore

@@ -48,6 +48,8 @@ test-search2
 test-search3
 test-search4
 test-search5
+test.ipynb
+
 # General
 .DS_Store
 

lamin_cli-1.2.0/.pre-commit-config.yaml

@@ -0,0 +1,40 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+        exclude: |
+          (?x)(
+              .github/workflows/latest-changes.jinja2
+          )
+      - id: check-yaml
+      - id: check-added-large-files
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.2
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix, --unsafe-fixes]
+      - id: ruff-format
+  - repo: https://github.com/rbubley/mirrors-prettier
+    rev: v3.5.1
+    hooks:
+      - id: prettier
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.3.9
+    hooks:
+      - id: nbstripout
+        exclude: |
+          (?x)(
+              docs/examples/|
+              docs/notes/|
+              tests
+          )
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.14.1
+    hooks:
+      - id: mypy
+        exclude: |
+          (?x)(
+              tests/hub-local/conftest.py
+          )

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamin_cli
-Version: 1.0.7
+Version: 1.2.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

lamin_cli-1.2.0/lamin_cli/__init__.py

@@ -0,0 +1,3 @@
+"""Lamin CLI."""
+
+__version__ = "1.2.0"

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/lamin_cli/__main__.py

@@ -1,12 +1,16 @@
 from __future__ import annotations
+
+import inspect
 import os
 import sys
+import warnings
 from collections import OrderedDict
-import inspect
-from importlib.metadata import PackageNotFoundError, version
-from typing import Optional, Mapping
 from functools import wraps
-import warnings
+from importlib.metadata import PackageNotFoundError, version
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
 
 # https://github.com/ewels/rich-click/issues/19
 # Otherwise rich-click takes over the formatting.
@@ -18,11 +22,11 @@ if os.environ.get("NO_RICH"):
 
         def __init__(
             self,
-            name: Optional[str] = None,
-            commands: Optional[Mapping[str, click.Command]] = None,
+            name: str | None = None,
+            commands: Mapping[str, click.Command] | None = None,
             **kwargs,
         ):
-            super(OrderedGroup, self).__init__(name, commands, **kwargs)
+            super().__init__(name, commands, **kwargs)
             self.commands = commands or OrderedDict()
 
         def list_commands(self, ctx: click.Context) -> Mapping[str, click.Command]:
@@ -77,12 +81,14 @@ else:
         return wrapper
 
 
-from click import Command, Context
 from lamindb_setup._silence_loggers import silence_loggers
 
-from lamin_cli._settings import settings
 from lamin_cli._cache import cache
 from lamin_cli._migration import migrate
+from lamin_cli._settings import settings
+
+if TYPE_CHECKING:
+    from click import Command, Context
 
 try:
     lamindb_version = version("lamindb")
@@ -100,7 +106,7 @@ def main():
 @main.command()
 @click.argument("user", type=str, default=None, required=False)
 @click.option("--key", type=str, default=None, help="The legacy API key.")
-def login(user: str, key: Optional[str]):
+def login(user: str, key: str | None):
     """Log into LaminHub.
 
     `lamin login` prompts for your API key unless you set it via environment variable `LAMIN_API_KEY`.
@@ -142,24 +148,25 @@ def schema_to_modules_callback(ctx, param, value):
             "The --schema option is deprecated and will be removed in a future version."
             " Please use --modules instead.",
             DeprecationWarning,
+            stacklevel=2,
         )
     return value
 
 
 # fmt: off
 @main.command()
-@click.option("--storage", type=str, help="Local directory, s3://bucket_name, gs://bucket_name.")  # noqa: E501
-@click.option("--db", type=str, default=None, help="Postgres database connection URL, do not pass for SQLite.")  # noqa: E501
-@click.option("--modules", type=str, default=None, help="Comma-separated string of modules.")  # noqa: E501
+@click.option("--storage", type=str, help="Local directory, s3://bucket_name, gs://bucket_name.")
+@click.option("--db", type=str, default=None, help="Postgres database connection URL, do not pass for SQLite.")
+@click.option("--modules", type=str, default=None, help="Comma-separated string of schema modules.")
 @click.option("--name", type=str, default=None, help="The instance name.")
-@click.option("--schema", type=str, default=None, help="[DEPRECATED] Use --modules instead.", callback=schema_to_modules_callback)  # noqa: E501
+@click.option("--schema", type=str, default=None, help="[DEPRECATED] Use --modules instead.", callback=schema_to_modules_callback)
 # fmt: on
 def init(
     storage: str,
-    db: Optional[str],
-    modules: Optional[str],
-    name: Optional[str],
-    schema: Optional[str],
+    db: str | None,
+    modules: str | None,
+    name: str | None,
+    schema: str | None,
 ):
     """Init an instance."""
     from lamindb_setup._init_instance import init as init_
@@ -182,7 +189,8 @@ def connect(instance: str):
     {attr}`~lamindb.setup.core.SetupSettings.auto_connect` to `True` so that you
     auto-connect in a Python session upon importing `lamindb`.
     """
-    from lamindb_setup import settings as settings_, connect as connect_
+    from lamindb_setup import connect as connect_
+    from lamindb_setup import settings as settings_
 
     settings_.auto_connect = True
     return connect_(instance, _reload_lamindb=False)
@@ -217,7 +225,7 @@ def info(schema: bool):
 # fmt: off
 @main.command()
 @click.argument("instance", type=str, default=None)
-@click.option("--force", is_flag=True, default=False, help="Do not ask for confirmation.")  # noqa: E501
+@click.option("--force", is_flag=True, default=False, help="Do not ask for confirmation.")
 # fmt: on
 def delete(instance: str, force: bool = False):
     """Delete an entity.
@@ -236,7 +244,7 @@ def delete(instance: str, force: bool = False):
 @click.option(
     "--with-env", is_flag=True, help="Also return the environment for a tranform."
 )
-def load(entity: str, uid: str = None, key: str = None, with_env: bool = False):
+def load(entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False):
     """Load a file or folder.
 
     Pass a URL, `artifact`, or `transform`. For example:
@@ -252,7 +260,8 @@ def load(entity: str, uid: str = None, key: str = None, with_env: bool = False):
     """
     is_slug = entity.count("/") == 1
     if is_slug:
-        from lamindb_setup import settings as settings_, connect
+        from lamindb_setup import connect
+        from lamindb_setup import settings as settings_
 
         # can decide whether we want to actually deprecate
         # click.echo(
@@ -270,18 +279,31 @@ def load(entity: str, uid: str = None, key: str = None, with_env: bool = False):
 @click.argument("entity", type=str)
 @click.option("--uid", help="The uid for the entity.")
 @click.option("--key", help="The key for the entity.")
-@click.option(
-    "--with-env", is_flag=True, help="Also return the environment for a tranform."
-)
-def get(entity: str, uid: str = None, key: str = None, with_env: bool = False):
+def get(entity: str, uid: str | None = None, key: str | None = None):
     """Query metadata about an entity.
 
-    Currently only works for artifact & transform and behaves like `lamin load`.
+    Currently only works for artifact.
     """
-    from lamin_cli._load import load as load_
+    import lamindb_setup as ln_setup
 
-    click.echo(f"! to load a file or folder, please use: lamin load {entity}")
-    return load_(entity, uid=uid, key=key, with_env=with_env)
+    from ._load import decompose_url
+
+    if entity.startswith("https://") and "lamin" in entity:
+        url = entity
+        instance, entity, uid = decompose_url(url)
+    elif entity not in {"artifact"}:
+        raise SystemExit("Entity has to be a laminhub URL or 'artifact'")
+    else:
+        instance = ln_setup.settings.instance.slug
+
+    ln_setup.connect(instance)
+    import lamindb as ln
+
+    if uid is not None:
+        artifact = ln.Artifact.get(uid)
+    else:
+        artifact = ln.Artifact.get(key=key)
+    artifact.describe()
 
 
 @main.command()
@@ -315,7 +337,7 @@ def _generate_help():
     out: dict[str, dict[str, str | None]] = {}
 
     def recursive_help(
-        cmd: Command, parent: Optional[Context] = None, name: tuple[str, ...] = ()
+        cmd: Command, parent: Context | None = None, name: tuple[str, ...] = ()
     ):
         ctx = click.Context(cmd, info_name=cmd.name, parent=parent)
         assert cmd.name

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/lamin_cli/_cache.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 import os
 
 if os.environ.get("NO_RICH"):

lamin_cli-1.2.0/lamin_cli/_load.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import re
+import shutil
+from pathlib import Path
+
+from lamin_utils import logger
+
+
+def decompose_url(url: str) -> tuple[str, str, str]:
+    assert any(keyword in url for keyword in ["transform", "artifact", "collection"])
+    for entity in ["transform", "artifact", "collection"]:
+        if entity in url:
+            break
+    uid = url.split(f"{entity}/")[1]
+    instance_slug = "/".join(url.split("/")[3:5])
+    return instance_slug, entity, uid
+
+
+def load(
+    entity: str, uid: str | None = None, key: str | None = None, with_env: bool = False
+):
+    import lamindb_setup as ln_setup
+
+    if entity.startswith("https://") and "lamin" in entity:
+        url = entity
+        instance, entity, uid = decompose_url(url)
+    elif entity not in {"artifact", "transform", "collection"}:
+        raise SystemExit(
+            "Entity has to be a laminhub URL or 'artifact', 'collection', or 'transform'"
+        )
+    else:
+        instance = ln_setup.settings.instance.slug
+
+    ln_setup.connect(instance)
+    import lamindb as ln
+
+    def script_to_notebook(
+        transform: ln.Transform, notebook_path: Path, bump_revision: bool = False
+    ) -> None:
+        import jupytext
+        from lamin_utils._base62 import increment_base62
+
+        if notebook_path.suffix == ".ipynb":
+            # below is backward compat
+            if "# # transform.name" in transform.source_code:
+                new_content = transform.source_code.replace(
+                    "# # transform.name", f"# # {transform.description}"
+                )
+            elif transform.source_code.startswith("# %% [markdown]"):
+                source_code_split = transform.source_code.split("\n")
+                if source_code_split[1] == "#":
+                    source_code_split[1] = f"# # {transform.description}"
+                new_content = "\n".join(source_code_split)
+            else:
+                new_content = transform.source_code
+        else:  # R notebook
+            # Pattern to match title only within YAML header section
+            title_pattern = r'^---\n.*?title:\s*"([^"]*)".*?---'
+            title_match = re.search(
+                title_pattern, transform.source_code, flags=re.DOTALL | re.MULTILINE
+            )
+            new_content = transform.source_code
+            if title_match:
+                current_title = title_match.group(1)
+                if current_title != transform.description:
+                    pattern = r'^(---\n.*?title:\s*)"([^"]*)"(.*?---)'
+                    replacement = f'\\1"{transform.description}"\\3'
+                    new_content = re.sub(
+                        pattern,
+                        replacement,
+                        new_content,
+                        flags=re.DOTALL | re.MULTILINE,
+                    )
+                    logger.important(
+                        f"updated title to match description: {current_title} →"
+                        f" {transform.description}"
+                    )
+        if bump_revision:
+            uid = transform.uid
+            if uid in new_content:
+                new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
+                new_content = new_content.replace(uid, new_uid)
+                logger.important(f"updated uid: {uid} → {new_uid}")
+        if notebook_path.suffix == ".ipynb":
+            notebook = jupytext.reads(new_content, fmt="py:percent")
+            jupytext.write(notebook, notebook_path)
+        else:
+            notebook_path.write_text(new_content)
+
+    query_by_uid = uid is not None
+
+    match entity:
+        case "transform":
+            if query_by_uid:
+                # we don't use .get here because DoesNotExist is hard to catch
+                # due to private django API
+                # here full uid is not expected anymore as before
+                # via ln.Transform.objects.get(uid=uid)
+                transforms = ln.Transform.objects.filter(uid__startswith=uid)
+            else:
+                # if below, we take is_latest=True as the criterion, we might get draft notebooks
+                # hence, we use source_code__isnull=False and order by created_at instead
+                transforms = ln.Transform.objects.filter(
+                    key=key, source_code__isnull=False
+                )
+
+            if (n_transforms := len(transforms)) == 0:
+                err_msg = f"uid {uid}" if query_by_uid else f"key={key} and source_code"
+                raise SystemExit(f"Transform with {err_msg} does not exist.")
+
+            if n_transforms > 1:
+                transforms = transforms.order_by("-created_at")
+            transform = transforms.first()
+
+            target_relpath = Path(transform.key)
+            if len(target_relpath.parents) > 1:
+                logger.important(
+                    "preserve the folder structure for versioning:"
+                    f" {target_relpath.parent}/"
+                )
+                target_relpath.parent.mkdir(parents=True, exist_ok=True)
+            if target_relpath.exists():
+                response = input(f"! {target_relpath} exists: replace? (y/n)")
+                if response != "y":
+                    raise SystemExit("Aborted.")
+
+            if transform.source_code is not None:
+                if target_relpath.suffix in (".ipynb", ".Rmd", ".qmd"):
+                    script_to_notebook(transform, target_relpath, bump_revision=True)
+                else:
+                    target_relpath.write_text(transform.source_code)
+            else:
+                raise SystemExit("No source code available for this transform.")
+
+            logger.important(f"{transform.type} is here: {target_relpath}")
+
+            if with_env:
+                ln.settings.track_run_inputs = False
+                if (
+                    transform.latest_run is not None
+                    and transform.latest_run.environment is not None
+                ):
+                    filepath_env_cache = transform.latest_run.environment.cache()
+                    target_env_filename = (
+                        target_relpath.parent
+                        / f"{target_relpath.stem}__requirements.txt"
+                    )
+                    shutil.move(filepath_env_cache, target_env_filename)
+                    logger.important(f"environment is here: {target_env_filename}")
+                else:
+                    logger.warning(
+                        "latest transform run with environment doesn't exist"
+                    )
+
+            return target_relpath
+        case "artifact" | "collection":
+            ln.settings.track_run_inputs = False
+
+            EntityClass = ln.Artifact if entity == "artifact" else ln.Collection
+
+            # we don't use .get here because DoesNotExist is hard to catch
+            # due to private django API
+            if query_by_uid:
+                entities = EntityClass.filter(uid__startswith=uid)
+            else:
+                entities = EntityClass.filter(key=key)
+
+            if (n_entities := len(entities)) == 0:
+                err_msg = f"uid={uid}" if query_by_uid else f"key={key}"
+                raise SystemExit(
+                    f"{entity.capitalize()} with {err_msg} does not exist."
+                )
+
+            if n_entities > 1:
+                entities = entities.order_by("-created_at")
+
+            entity_obj = entities.first()
+            cache_path = entity_obj.cache()
+
+            # collection gives us a list of paths
+            if isinstance(cache_path, list):
+                logger.important(f"{entity} paths ({len(cache_path)} files):")
+                for i, path in enumerate(cache_path):
+                    if i < 5 or i >= len(cache_path) - 5:
+                        logger.important(f"  [{i + 1}/{len(cache_path)}] {path}")
+                    elif i == 5:
+                        logger.important(f"  ... {len(cache_path) - 10} more files ...")
+            else:
+                logger.important(f"{entity} is here: {cache_path}")
+        case _:
+            raise AssertionError(f"unknown entity {entity}")

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/lamin_cli/_migration.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 import os
 from typing import Optional
 
@@ -34,9 +35,9 @@ def deploy():
 @click.option("--end-number", type=str, default=None)
 @click.option("--start-number", type=str, default=None)
 def squash(
-    package_name: Optional[str],
-    end_number: Optional[str],
-    start_number: Optional[str],
+    package_name: str | None,
+    end_number: str | None,
+    start_number: str | None,
 ):
     """Squash migrations."""
     from lamindb_setup._migrate import migrate

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/lamin_cli/_save.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
+
+import re
+import sys
 from pathlib import Path
-from typing import Union
+
 from lamin_utils import logger
-import re
 
 
 def parse_uid_from_code(content: str, suffix: str) -> str | None:
@@ -41,7 +43,7 @@ def parse_uid_from_code(content: str, suffix: str) -> str | None:
 
 
 def save_from_filepath_cli(
-    filepath: Union[str, Path],
+    filepath: str | Path,
     key: str | None,
     description: str | None,
     registry: str | None,
@@ -57,13 +59,16 @@ def save_from_filepath_cli(
     ln_setup.settings.auto_connect = True
 
     import lamindb as ln
+
+    if not ln.setup.core.django.IS_SETUP:
+        sys.exit(-1)
     from lamindb._finish import save_context_core
 
     ln_setup.settings.auto_connect = auto_connect_state
 
     suffixes_transform = {
-        "py": set([".py", ".ipynb"]),
-        "R": set([".R", ".qmd", ".Rmd"]),
+        "py": {".py", ".ipynb"},
+        "R": {".R", ".qmd", ".Rmd"},
     }
 
     if filepath.suffix in {".qmd", ".Rmd"}:
@@ -80,8 +85,8 @@ def save_from_filepath_cli(
             and filepath.with_suffix(".nb.html").exists()
         ):
             raise SystemExit(
-                f'Please delete one of\n - {filepath.with_suffix(".html")}\n -'
-                f' {filepath.with_suffix(".nb.html")}'
+                f"Please delete one of\n - {filepath.with_suffix('.html')}\n -"
+                f" {filepath.with_suffix('.nb.html')}"
             )
 
     if registry is None:

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/lamin_cli/_settings.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 import os
 
 if os.environ.get("NO_RICH"):

lamin_cli-1.2.0/pyproject.toml

@@ -0,0 +1,114 @@
+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "lamin_cli"
+authors = [{name = "Lamin Labs", email = "open-source@lamin.ai"}]
+readme = "README.md"
+dynamic = ["version", "description"]
+dependencies = [
+    "rich-click>=1.7",
+]
+
+[project.urls]
+Home = "https://github.com/laminlabs/lamin-cli"
+
+[project.scripts]
+lamin = "lamin_cli.__main__:main"
+
+[tool.ruff]
+src = ["src"]
+line-length = 88
+lint.select = [
+    "F",  # Errors detected by Pyflakes
+    "E",  # Error detected by Pycodestyle
+    "W",  # Warning detected by Pycodestyle
+    "I",  # isort
+    "D",  # pydocstyle
+    "B",  # flake8-bugbear
+    "TID",  # flake8-tidy-imports
+    "C4",  # flake8-comprehensions
+    "BLE",  # flake8-blind-except
+    "UP",  # pyupgrade
+    "RUF100",  # Report unused noqa directives
+    "TCH",  # Typing imports
+    "NPY",  # Numpy specific rules
+    "PTH"  # Use pathlib
+]
+lint.ignore = [
+    # Do not catch blind exception: `Exception`
+    "BLE001",
+    # Errors from function calls in argument defaults. These are fine when the result is immutable.
+    "B008",
+    # line too long -> we accept long comment lines; black gets rid of long code lines
+    "E501",
+    # Do not assign a lambda expression, use a def -> lambda expression assignments are convenient
+    "E731",
+    # allow I, O, l as variable names -> I is the identity matrix
+    "E741",
+    # Missing docstring in public module
+    "D100",
+    # undocumented-public-class
+    "D101",
+    # Missing docstring in public method
+    "D102",
+    # Missing docstring in public function
+    "D103",
+    # Missing docstring in public package
+    "D104",
+    # __magic__ methods are are often self-explanatory, allow missing docstrings
+    "D105",
+    # Missing docstring in public nested class
+    "D106",
+    # Missing docstring in __init__
+    "D107",
+    ## Disable one in each pair of mutually incompatible rules
+    # We don’t want a blank line before a class docstring
+    "D203",
+    # 1 blank line required after class docstring
+    "D204",
+    # first line should end with a period [Bug: doesn't work with single-line docstrings]
+    # We want docstrings to start immediately after the opening triple quote
+    "D213",
+    # Section underline is over-indented ("{name}")
+    "D215",
+    # First line should end with a period
+    "D400",
+    # First line should be in imperative mood; try rephrasing
+    "D401",
+    # First word of the first line should be capitalized: {} -> {}
+    "D403",
+    # First word of the docstring should not be "This"
+    "D404",
+    # Section name should end with a newline ("{name}")
+    "D406",
+    # Missing dashed underline after section ("{name}")
+    "D407",
+    # Section underline should be in the line following the section's name ("{name}")
+    "D408",
+    # Section underline should match the length of its name ("{name}")
+    "D409",
+    # No blank lines allowed between a section header and its content ("{name}")
+    "D412",
+    # Missing blank line after last section ("{name}")
+    "D413",
+    # Missing argument description
+    "D417",
+    # Imports unused
+    "F401",
+    # camcelcase imported as lowercase
+    "N813",
+    # module import not at top level of file
+    "E402",
+    # open()` should be replaced by `Path.open()
+    "PTH123",
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+"docs/*" = ["I", "B018", "B017"]
+"tests/*" = ["D"]
+"*/__init__.py" = ["F401"]

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/scripts/merely-import-lamindb.py

@@ -1,4 +1,4 @@
-import lamindb as ln  # noqa
+import lamindb as ln
 
 if __name__ == "__main__":
     print("hello!")

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/scripts/run-track-and-finish-sync-git.py

@@ -1,7 +1,7 @@
 import lamindb as ln
 
 ln.settings.sync_git_repo = "https://github.com/laminlabs/lamin-cli"
-ln.context.name = "My good script"
+ln.context.description = "My good script"
 ln.track("m5uCHTTpJnjQ0000")
 
 

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/scripts/run-track-with-params.py

@@ -1,4 +1,5 @@
 import argparse
+
 import lamindb as ln
 
 if __name__ == "__main__":

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/test_cli.py

@@ -1,5 +1,6 @@
 import os
 from datetime import datetime, timedelta, timezone
+
 from lamindb_setup import settings
 from lamindb_setup.core._hub_client import connect_hub_with_auth
 from lamindb_setup.core._hub_core import create_api_key

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/test_load.py

@@ -1,12 +1,13 @@
-from lamin_cli._load import decompose_url
-from pathlib import Path
 import subprocess
+from pathlib import Path
+
+from lamin_cli._load import decompose_url
 
 
 def test_decompose_url():
     urls = [
-        "https://lamin.ai/laminlabs/arrayloader-benchmarks/transform/1GCKs8zLtkc85zKv",  # noqa
-        "https://lamin.company.com/laminlabs/arrayloader-benchmarks/transform/1GCKs8zLtkc85zKv",  # noqa
+        "https://lamin.ai/laminlabs/arrayloader-benchmarks/transform/1GCKs8zLtkc85zKv",
+        "https://lamin.company.com/laminlabs/arrayloader-benchmarks/transform/1GCKs8zLtkc85zKv",
     ]
     for url in urls:
         result = decompose_url(url)
@@ -23,7 +24,7 @@ def test_load_transform():
     result = subprocess.run(
         "lamin load"
         " 'https://lamin.ai/laminlabs/lamin-dev/transform/VFYCIuaw2GsX0000'"
-        " --with-env",  # noqa
+        " --with-env",
         shell=True,
         capture_output=True,
     )
@@ -59,10 +60,18 @@ def test_load_transform():
     path2.unlink()
 
 
-def test_load_artifact():
+def test_get_load_artifact():
+    result = subprocess.run(
+        "lamin get"
+        " 'https://lamin.ai/laminlabs/lamin-site-assets/artifact/e2G7k9EVul4JbfsEYAy5'",
+        shell=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0
+
     result = subprocess.run(
         "lamin load"
-        " 'https://lamin.ai/laminlabs/lamin-site-assets/artifact/e2G7k9EVul4JbfsEYAy5'",  # noqa
+        " 'https://lamin.ai/laminlabs/lamin-site-assets/artifact/e2G7k9EVul4JbfsEYAy5'",
         shell=True,
         capture_output=True,
     )
@@ -75,3 +84,12 @@ def test_load_artifact():
         capture_output=True,
     )
     assert result.returncode == 0
+
+
+def test_load_collection():
+    result = subprocess.run(
+        "lamin load 'https://lamin.ai/laminlabs/lamindata/collection/2wUs6V1OuGzp5Ll4'",
+        shell=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/test_parse_uid_from_code.py

@@ -112,6 +112,6 @@ def test_r_track_pattern():
     for suffix in suffixes:
         for content, expected_uid in valid_cases:
             uid = parse_uid_from_code(content, suffix)
-            assert (
-                uid == expected_uid
-            ), f"Failed for valid content with {suffix}: {content}"
+            assert uid == expected_uid, (
+                f"Failed for valid content with {suffix}: {content}"
+            )

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/test_save_files.py

@@ -1,7 +1,8 @@
 import subprocess
-import lamindb_setup as ln_setup
 from pathlib import Path
 
+import lamindb_setup as ln_setup
+
 test_file = Path(__file__).parent.parent.resolve() / ".gitignore"
 
 

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/test_save_notebooks.py

@@ -1,12 +1,13 @@
+import json
 import os
 import subprocess
 from pathlib import Path
+
+import lamindb as ln
 import nbproject_test
 import pytest
-from nbproject.dev import read_notebook, write_notebook
 from nbclient.exceptions import CellExecutionError
-import json
-import lamindb as ln
+from nbproject.dev import read_notebook, write_notebook
 
 notebook_dir = "./sub/lamin-cli/tests/notebooks/"
 
@@ -167,13 +168,13 @@ print("my consecutive cell")
     # get the the source code via command line
     result = subprocess.run(
         "yes | lamin load"
-        f" https://lamin.ai/{ln.setup.settings.user.handle}/laminci-unit-tests/transform/hlsFXswrJjtt0000",  # noqa
+        f" https://lamin.ai/{ln.setup.settings.user.handle}/laminci-unit-tests/transform/hlsFXswrJjtt0000",
         shell=True,
         capture_output=True,
     )
     # print(result.stderr.decode())
     assert Path("./with-title-and-initialized-consecutive.ipynb").exists()
-    with open("./with-title-and-initialized-consecutive.ipynb", "r") as f:
+    with open("./with-title-and-initialized-consecutive.ipynb") as f:
         json_notebook = json.load(f)
     print(json_notebook["cells"][0])
     assert json_notebook["cells"][0]["source"] == ["# My test notebook (consecutive)"]

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/test_save_r_code.py

@@ -1,6 +1,7 @@
-from pathlib import Path
-import subprocess
 import os
+import subprocess
+from pathlib import Path
+
 import lamindb as ln
 
 scripts_dir = Path(__file__).parent.resolve() / "scripts"

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/test_save_scripts.py

@@ -1,10 +1,10 @@
-from pathlib import Path
-import subprocess
 import os
+import subprocess
+from pathlib import Path
+
 import lamindb as ln
 from lamindb_setup import settings
 
-
 scripts_dir = Path(__file__).parent.resolve() / "scripts"
 
 
@@ -45,17 +45,38 @@ def test_run_save_cache_with_git_and_uid():
         shell=True,
         capture_output=True,
     )
-    # print(result.stdout.decode())
-    # print(result.stderr.decode())
+    print(result.stdout.decode())
+    print(result.stderr.decode())
     assert result.returncode == 0
     assert "created Transform" in result.stdout.decode()
     assert "m5uCHTTp" in result.stdout.decode()
     assert "started new Run" in result.stdout.decode()
 
     transform = ln.Transform.get("m5uCHTTpJnjQ")
-    assert transform.hash == "MoIciBQ0lpVPCKQGofPX6g"
+    assert transform.hash == "VC1oTPcaVSrzNrXUT9p4qw"
     assert transform.latest_run.environment.path.exists()
 
+    assert (
+        transform.source_code
+        == """import lamindb as ln
+
+ln.settings.sync_git_repo = "https://github.com/laminlabs/lamin-cli"
+ln.context.description = "My good script"
+ln.track("m5uCHTTpJnjQ0000")
+
+
+if __name__ == "__main__":
+    # we're using new_run here to mock the notebook situation
+    # and cover the look up of an existing run in the tests
+    # new_run = True is trivial
+    ln.track(new_run=False)
+
+    print("hello!")
+
+    ln.finish()
+"""
+    )
+
     # you can rerun the same script
     result = subprocess.run(
         f"python {filepath}",
@@ -160,7 +181,7 @@ def test_run_save_cache_with_git_and_uid():
     # try to get the the source code via command line
     result = subprocess.run(
         "yes | lamin load"
-        f" https://lamin.ai/{settings.user.handle}/laminci-unit-tests/transform/m5uCHTTpJnjQ0000",  # noqa
+        f" https://lamin.ai/{settings.user.handle}/laminci-unit-tests/transform/m5uCHTTpJnjQ0000",
         shell=True,
         capture_output=True,
     )
@@ -168,7 +189,7 @@ def test_run_save_cache_with_git_and_uid():
     assert result.returncode == 0
 
     result = subprocess.run(
-        f"yes | lamin load transform --key {filepath.name}",  # noqa
+        f"yes | lamin load transform --key {filepath.name}",
         shell=True,
         capture_output=True,
     )

lamin_cli-1.0.7/.pre-commit-config.yaml

@@ -1,64 +0,0 @@
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-        exclude: |
-          (?x)(
-              .github/workflows/latest-changes.jinja2
-          )
-      - id: check-yaml
-      - id: check-added-large-files
-  - repo: https://github.com/psf/black
-    rev: 22.6.0
-    hooks:
-      - id: black-jupyter
-  - repo: https://github.com/pycqa/flake8
-    rev: 4.0.1
-    hooks:
-      - id: flake8
-        additional_dependencies:
-          - flake8-black==0.3.3
-          - flake8-typing-imports==1.10.0
-        language_version: python3
-        args:
-          - --max-line-length=120
-          - --ignore=E203,W503,BLK100,TYP001
-        exclude: |
-          (?x)(
-              __init__.py
-          )
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.6.2
-    hooks:
-      - id: prettier
-  - repo: https://github.com/kynan/nbstripout
-    rev: 0.3.9
-    hooks:
-      - id: nbstripout
-        exclude: |
-          (?x)(
-              docs/examples/|
-              docs/notes/|
-              tests
-          )
-  - repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.1.9
-    hooks:
-      - id: forbid-crlf
-      - id: remove-crlf
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.940
-    hooks:
-      - id: mypy
-        exclude: |
-          (?x)(
-              tests/hub-local/conftest.py
-          )
-  - repo: https://github.com/pycqa/pydocstyle
-    rev: 6.1.1
-    hooks:
-      - id: pydocstyle
-        args: # google style + __init__, see http://www.pydocstyle.org/en/stable/error_codes.html
-          - --ignore=D100,D101,D102,D103,D104,D106,D107,D203,D204,D213,D215,D400,D401,D403,D404,D406,D407,D408,D409,D412,D413

lamin_cli-1.0.7/lamin_cli/__init__.py

@@ -1,3 +0,0 @@
-"""Lamin CLI."""
-
-__version__ = "1.0.7"

lamin_cli-1.0.7/lamin_cli/_load.py

@@ -1,164 +0,0 @@
-from __future__ import annotations
-from typing import Tuple
-from lamin_utils import logger
-import shutil
-import re
-from pathlib import Path
-
-
-def decompose_url(url: str) -> Tuple[str, str, str]:
-    assert "transform" in url or "artifact" in url
-    for entity in ["transform", "artifact"]:
-        if entity in url:
-            break
-    uid = url.split(f"{entity}/")[1]
-    instance_slug = "/".join(url.split("/")[3:5])
-    return instance_slug, entity, uid
-
-
-def load(entity: str, uid: str = None, key: str = None, with_env: bool = False):
-    import lamindb_setup as ln_setup
-
-    if entity.startswith("https://") and "lamin" in entity:
-        url = entity
-        instance, entity, uid = decompose_url(url)
-    elif entity not in {"artifact", "transform"}:
-        raise SystemExit("Entity has to be a laminhub URL or 'artifact' or 'transform'")
-    else:
-        instance = ln_setup.settings.instance.slug
-
-    ln_setup.connect(instance)
-    import lamindb as ln
-
-    def script_to_notebook(
-        transform: ln.Transform, notebook_path: Path, bump_revision: bool = False
-    ) -> None:
-        import jupytext
-        from lamin_utils._base62 import increment_base62
-
-        if notebook_path.suffix == ".ipynb":
-            # below is backward compat
-            if "# # transform.name" in transform.source_code:
-                new_content = transform.source_code.replace(
-                    "# # transform.name", f"# # {transform.description}"
-                )
-            elif transform.source_code.startswith("# %% [markdown]\n#\n"):
-                new_content = transform.source_code.replace(
-                    "# %% [markdown]\n#\n",
-                    f"# %% [markdown]\n# # {transform.description}\n",
-                )
-        else:  # R notebook
-            # Pattern to match title only within YAML header section
-            title_pattern = r'^---\n.*?title:\s*"([^"]*)".*?---'
-            title_match = re.search(
-                title_pattern, transform.source_code, flags=re.DOTALL | re.MULTILINE
-            )
-            new_content = transform.source_code
-            if title_match:
-                current_title = title_match.group(1)
-                if current_title != transform.description:
-                    pattern = r'^(---\n.*?title:\s*)"([^"]*)"(.*?---)'
-                    replacement = f'\\1"{transform.description}"\\3'
-                    new_content = re.sub(
-                        pattern,
-                        replacement,
-                        new_content,
-                        flags=re.DOTALL | re.MULTILINE,
-                    )
-                    logger.important(
-                        f"updated title to match description: {current_title} →"
-                        f" {transform.description}"
-                    )
-        if bump_revision:
-            uid = transform.uid
-            if uid in new_content:
-                new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
-                new_content = new_content.replace(uid, new_uid)
-                logger.important(f"updated uid: {uid} → {new_uid}")
-        if notebook_path.suffix == ".ipynb":
-            notebook = jupytext.reads(new_content, fmt="py:percent")
-            jupytext.write(notebook, notebook_path)
-        else:
-            notebook_path.write_text(new_content)
-
-    query_by_uid = uid is not None
-
-    if entity == "transform":
-        if query_by_uid:
-            # we don't use .get here because DoesNotExist is hard to catch
-            # due to private django API
-            # here full uid is not expected anymore as before
-            # via ln.Transform.objects.get(uid=uid)
-            transforms = ln.Transform.objects.filter(uid__startswith=uid)
-        else:
-            # if below, we take is_latest=True as the criterion, we might get draft notebooks
-            # hence, we use source_code__isnull=False and order by created_at instead
-            transforms = ln.Transform.objects.filter(key=key, source_code__isnull=False)
-
-        if (n_transforms := len(transforms)) == 0:
-            err_msg = f"uid {uid}" if query_by_uid else f"key={key} and source_code"
-            raise SystemExit(f"Transform with {err_msg} does not exist.")
-
-        if n_transforms > 1:
-            transforms = transforms.order_by("-created_at")
-        transform = transforms.first()
-
-        target_relpath = Path(transform.key)
-        if len(target_relpath.parents) > 1:
-            logger.important(
-                "preserve the folder structure for versioning:"
-                f" {target_relpath.parent}/"
-            )
-            target_relpath.parent.mkdir(parents=True, exist_ok=True)
-        if target_relpath.exists():
-            response = input(f"! {target_relpath} exists: replace? (y/n)")
-            if response != "y":
-                raise SystemExit("Aborted.")
-
-        if transform.source_code is not None:
-            if target_relpath.suffix in (".ipynb", ".Rmd", ".qmd"):
-                script_to_notebook(transform, target_relpath, bump_revision=True)
-            else:
-                target_relpath.write_text(transform.source_code)
-        else:
-            raise SystemExit("No source code available for this transform.")
-
-        logger.important(f"{transform.type} is here: {target_relpath}")
-
-        if with_env:
-            ln.settings.track_run_inputs = False
-            if (
-                transform.latest_run is not None
-                and transform.latest_run.environment is not None
-            ):
-                filepath_env_cache = transform.latest_run.environment.cache()
-                target_env_filename = (
-                    target_relpath.parent / f"{target_relpath.stem}__requirements.txt"
-                )
-                shutil.move(filepath_env_cache, target_env_filename)
-                logger.important(f"environment is here: {target_env_filename}")
-            else:
-                logger.warning("latest transform run with environment doesn't exist")
-
-        return target_relpath
-    elif entity == "artifact":
-        ln.settings.track_run_inputs = False
-
-        if query_by_uid:
-            # we don't use .get here because DoesNotExist is hard to catch
-            # due to private django API
-            artifacts = ln.Artifact.filter(uid__startswith=uid)
-        else:
-            artifacts = ln.Artifact.filter(key=key)
-
-        if (n_artifacts := len(artifacts)) == 0:
-            err_msg = f"uid={uid}" if query_by_uid else f"key={key}"
-            raise SystemExit(f"Artifact with {err_msg} does not exist.")
-
-        if n_artifacts > 1:
-            artifacts = artifacts.order_by("-created_at")
-        artifact = artifacts.first()
-
-        cache_path = artifact.cache()
-        logger.important(f"artifact is here: {cache_path}")
-        return cache_path

lamin_cli-1.0.7/pyproject.toml

@@ -1,21 +0,0 @@
-[build-system]
-requires = ["flit_core >=3.2,<4"]
-build-backend = "flit_core.buildapi"
-
-[project]
-name = "lamin_cli"
-authors = [{name = "Lamin Labs", email = "open-source@lamin.ai"}]
-readme = "README.md"
-dynamic = ["version", "description"]
-dependencies = [
-    "rich-click>=1.7",
-]
-
-[project.urls]
-Home = "https://github.com/laminlabs/lamin-cli"
-
-[project.scripts]
-lamin = "lamin_cli.__main__:main"
-
-[tool.black]
-preview = true

{lamin_cli-1.0.7 → lamin_cli-1.2.0}/tests/test_multi_process.py

@@ -1,6 +1,6 @@
+import subprocess
 from multiprocessing import Process
 from pathlib import Path
-import subprocess
 
 scripts_dir = Path(__file__).parent.resolve() / "scripts"