lamin_cli 1.3.0__py2.py3-none-any.whl → 1.4.0__py2.py3-none-any.whl

lamin_cli/__init__.py

@@ -1,3 +1,3 @@
 """Lamin CLI."""
 
-__version__ = "1.3.0"
+__version__ = "1.4.0"

lamin_cli/__main__.py

@@ -2,11 +2,13 @@ from __future__ import annotations
 
 import inspect
 import os
+import shutil
 import sys
 import warnings
 from collections import OrderedDict
 from functools import wraps
 from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 from lamindb_setup._init_instance import (
@@ -307,21 +309,28 @@ def get(entity: str, uid: str | None = None, key: str | None = None):
 
 
 @main.command()
-@click.argument("path", type=click.Path(exists=True, dir_okay=True, file_okay=True))
-@click.option("--key", type=str, default=None)
-@click.option("--description", type=str, default=None)
-@click.option("--stem-uid", type=str, default=None)
-@click.option("--registry", type=str, default=None)
-def save(path: str, key: str, description: str, stem_uid: str, registry: str):
+@click.argument("path", type=str)
+@click.option("--key", type=str, default=None, help="The key of the artifact or transform.")
+@click.option("--description", type=str, default=None, help="A description of the artifact or transform.")
+@click.option("--stem-uid", type=str, default=None, help="The stem uid of the artifact or transform.")
+@click.option("--project", type=str, default=None, help="A valid project name or uid.")
+@click.option("--registry", type=str, default=None, help="Either 'artifact' or 'transform'. If not passed, chooses based on path suffix.")
+def save(path: str, key: str, description: str, stem_uid: str, project: str, registry: str):
     """Save a file or folder.
 
-    Defaults to saving `.py`, `.ipynb`, `.R`, `.Rmd`, and `.qmd` as {class}`~lamindb.Transform` and
-    other file types and folders as {class}`~lamindb.Artifact`. You can save a `.py` or `.ipynb` file as
+    Example: Given a valid project name "my_project".
+
+    ```
+    lamin save my_table.csv --key my_tables/my_table.csv --project my_project
+    ```
+
+    Note: Defaults to saving `.py`, `.ipynb`, `.R`, `.Rmd`, and `.qmd` as {class}`~lamindb.Transform` and
+    other file types and folders as {class}`~lamindb.Artifact`. You can enforce saving a file as
     an {class}`~lamindb.Artifact` by passing `--registry artifact`.
     """
-    from lamin_cli._save import save_from_filepath_cli
+    from lamin_cli._save import save_from_path_cli
 
-    if save_from_filepath_cli(path, key, description, stem_uid, registry) is not None:
+    if save_from_path_cli(path, key, description, stem_uid, project, registry) is not None:
         sys.exit(1)
 
 
@@ -343,9 +352,6 @@ def run(filepath: str, project: str, image_url: str, packages: str, cpu: int, gp
     lamin run my_script.py --project my_project
     ```
     """
-    import shutil
-    from pathlib import Path
-
     from lamin_cli.compute.modal import Runner
 
     default_mount_dir = Path('./modal_mount_dir')
@@ -354,7 +360,7 @@ def run(filepath: str, project: str, image_url: str, packages: str, cpu: int, gp
 
     shutil.copy(filepath, default_mount_dir)
 
-    filepath_in_mount_dir = Path(default_mount_dir) / Path(filepath).name
+    filepath_in_mount_dir = default_mount_dir / Path(filepath).name
 
     package_list = []
     if packages:

lamin_cli/_load.py

@@ -78,7 +78,9 @@ def load(
                     )
         if bump_revision:
             uid = transform.uid
-            if uid in new_content:
+            if (
+                uid in new_content
+            ):  # this only hits if it has the full uid, not for the stem uid
                 new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
                 new_content = new_content.replace(uid, new_uid)
                 logger.important(f"updated uid: {uid} → {new_uid}")

lamin_cli/_save.py

@@ -2,26 +2,30 @@ from __future__ import annotations
 
 import re
 import sys
-from pathlib import Path
+from typing import TYPE_CHECKING
 
+import click
 from lamin_utils import logger
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
 
 def parse_uid_from_code(content: str, suffix: str) -> str | None:
     if suffix == ".py":
         track_pattern = re.compile(
-            r'ln\.track\(\s*(?:transform\s*=\s*)?(["\'])([a-zA-Z0-9]{16})\1'
+            r'ln\.track\(\s*(?:transform\s*=\s*)?(["\'])([a-zA-Z0-9]{12,16})\1'
         )
         uid_pattern = re.compile(r'\.context\.uid\s*=\s*["\']([^"\']+)["\']')
     elif suffix == ".ipynb":
         track_pattern = re.compile(
-            r'ln\.track\(\s*(?:transform\s*=\s*)?(?:\\"|\')([a-zA-Z0-9]{16})(?:\\"|\')'
+            r'ln\.track\(\s*(?:transform\s*=\s*)?(?:\\"|\')([a-zA-Z0-9]{12,16})(?:\\"|\')'
         )
         # backward compat
         uid_pattern = re.compile(r'\.context\.uid\s*=\s*\\["\']([^"\']+)\\["\']')
     elif suffix in {".R", ".qmd", ".Rmd"}:
         track_pattern = re.compile(
-            r'track\(\s*(?:transform\s*=\s*)?([\'"])([a-zA-Z0-9]{16})\1'
+            r'track\(\s*(?:transform\s*=\s*)?([\'"])([a-zA-Z0-9]{12,16})\1'
         )
         uid_pattern = None
     else:
@@ -42,17 +46,16 @@ def parse_uid_from_code(content: str, suffix: str) -> str | None:
     return uid
 
 
-def save_from_filepath_cli(
-    filepath: str | Path,
+def save_from_path_cli(
+    path: Path | str,
     key: str | None,
     description: str | None,
     stem_uid: str | None,
+    project: str | None,
     registry: str | None,
 ) -> str | None:
     import lamindb_setup as ln_setup
-
-    if not isinstance(filepath, Path):
-        filepath = Path(filepath)
+    from lamindb_setup.core.upath import LocalPathClasses, UPath, create_path
 
     # this will be gone once we get rid of lamin load or enable loading multiple
     # instances sequentially
@@ -67,37 +70,36 @@ def save_from_filepath_cli(
 
     ln_setup.settings.auto_connect = auto_connect_state
 
-    suffixes_transform = {
-        "py": {".py", ".ipynb"},
-        "R": {".R", ".qmd", ".Rmd"},
-    }
-
-    if filepath.suffix in {".qmd", ".Rmd"}:
-        if not (
-            filepath.with_suffix(".html").exists()
-            or filepath.with_suffix(".nb.html").exists()
-        ):
-            raise SystemExit(
-                f"Please export your {filepath.suffix} file as an html file here"
-                f" {filepath.with_suffix('.html')}"
-            )
-        if (
-            filepath.with_suffix(".html").exists()
-            and filepath.with_suffix(".nb.html").exists()
-        ):
-            raise SystemExit(
-                f"Please delete one of\n - {filepath.with_suffix('.html')}\n -"
-                f" {filepath.with_suffix('.nb.html')}"
-            )
+    # this allows to have the correct treatment of credentials in case of cloud paths
+    path = create_path(path)
+    # isinstance is needed to cast the type of path to UPath
+    # to avoid mypy erors
+    assert isinstance(path, UPath)
+    if not path.exists():
+        raise click.BadParameter(f"Path {path} does not exist", param_hint="path")
 
     if registry is None:
+        suffixes_transform = {
+            "py": {".py", ".ipynb"},
+            "R": {".R", ".qmd", ".Rmd"},
+        }
         registry = (
             "transform"
-            if filepath.suffix
-            in suffixes_transform["py"].union(suffixes_transform["R"])
+            if path.suffix in suffixes_transform["py"].union(suffixes_transform["R"])
             else "artifact"
         )
 
+    if project is not None:
+        project_record = ln.Project.filter(
+            ln.Q(name=project) | ln.Q(uid=project)
+        ).one_or_none()
+        if project_record is None:
+            raise ln.errors.InvalidArgument(
+                f"Project '{project}' not found, either create it with `ln.Project(name='...').save()` or fix typos."
+            )
+
+    is_cloud_path = not isinstance(path, LocalPathClasses)
+
     if registry == "artifact":
         ln.settings.creation.artifact_silence_missing_run_warning = True
         revises = None
@@ -109,56 +111,108 @@ def save_from_filepath_cli(
             )
             if revises is None:
                 raise ln.errors.InvalidArgument("The stem uid is not found.")
+
+        if is_cloud_path:
+            if key is not None:
+                logger.error("Do not pass --key for cloud paths")
+                return "key-with-cloud-path"
         elif key is None and description is None:
             logger.error("Please pass a key or description via --key or --description")
             return "missing-key-or-description"
+
         artifact = ln.Artifact(
-            filepath, key=key, description=description, revises=revises
+            path, key=key, description=description, revises=revises
         ).save()
         logger.important(f"saved: {artifact}")
         logger.important(f"storage path: {artifact.path}")
         if ln_setup.settings.storage.type == "s3":
             logger.important(f"storage url: {artifact.path.to_url()}")
+        if project is not None:
+            artifact.projects.add(project_record)
+            logger.important(f"labeled with project: {project_record.name}")
         if ln_setup.settings.instance.is_remote:
             slug = ln_setup.settings.instance.slug
             logger.important(f"go to: https://lamin.ai/{slug}/artifact/{artifact.uid}")
         return None
-    elif registry == "transform":
-        with open(filepath) as file:
-            content = file.read()
-        uid = parse_uid_from_code(content, filepath.suffix)
-        if uid is not None:
-            logger.important(f"mapped '{filepath}' on uid '{uid}'")
-            transform = ln.Transform.filter(uid=uid).one_or_none()
-            if transform is None:
+
+    if registry == "transform":
+        if is_cloud_path:
+            logger.error("Can not register a transform from a cloud path")
+            return "transform-with-cloud-path"
+
+        if path.suffix in {".qmd", ".Rmd"}:
+            html_file_exists = path.with_suffix(".html").exists()
+            nb_html_file_exists = path.with_suffix(".nb.html").exists()
+
+            if not html_file_exists and not nb_html_file_exists:
                 logger.error(
-                    f"Did not find uid '{uid}'"
-                    " in Transform registry. Did you run `ln.track()`?"
+                    f"Please export your {path.suffix} file as an html file here"
+                    f" {path.with_suffix('.html')}"
                 )
-                return "not-tracked-in-transform-registry"
-        else:
-            revises = None
-            if stem_uid is not None:
-                revises = (
-                    ln.Transform.filter(uid__startswith=stem_uid)
+                return "export-qmd-Rmd-as-html"
+            elif html_file_exists and nb_html_file_exists:
+                logger.error(
+                    f"Please delete one of\n - {path.with_suffix('.html')}\n -"
+                    f" {path.with_suffix('.nb.html')}"
+                )
+                return "delete-html-or-nb-html"
+
+        with path.open() as file:
+            content = file.read()
+        uid = parse_uid_from_code(content, path.suffix)
+
+        if uid is not None:
+            logger.important(f"mapped '{path.name}' on uid '{uid}'")
+            if len(uid) == 16:
+                # is full uid
+                transform = ln.Transform.filter(uid=uid).one_or_none()
+            else:
+                # is stem uid
+                if stem_uid is not None:
+                    assert stem_uid == uid, (
+                        "passed stem uid and parsed stem uid do not match"
+                    )
+                else:
+                    stem_uid = uid
+                transform = (
+                    ln.Transform.filter(uid__startswith=uid)
                     .order_by("-created_at")
                     .first()
                 )
-                if revises is None:
-                    raise ln.errors.InvalidArgument("The stem uid is not found.")
-            # TODO: build in the logic that queries for relative file paths
-            # we have in Context; add tests for multiple versions
-            transform = ln.Transform.filter(
-                key=filepath.name, is_latest=True
-            ).one_or_none()
-            if transform is None:
-                transform = ln.Transform(
-                    description=filepath.name,
-                    key=filepath.name,
-                    type="script" if filepath.suffix in {".R", ".py"} else "notebook",
-                    revises=revises,
-                ).save()
-                logger.important(f"created Transform('{transform.uid}')")
+                if transform is None:
+                    uid = f"{stem_uid}0000"
+        else:
+            # TODO: account for folders and hash equivalence as we do in ln.track()
+            transform = ln.Transform.filter(key=path.name, is_latest=True).one_or_none()
+        revises = None
+        if stem_uid is not None:
+            revises = (
+                ln.Transform.filter(uid__startswith=stem_uid)
+                .order_by("-created_at")
+                .first()
+            )
+            if revises is None:
+                raise ln.errors.InvalidArgument("The stem uid is not found.")
+        if transform is None:
+            if path.suffix == ".ipynb":
+                from nbproject.dev import read_notebook
+                from nbproject.dev._meta_live import get_title
+
+                nb = read_notebook(path)
+                description = get_title(nb)
+            else:
+                description = None
+            transform = ln.Transform(
+                uid=uid,
+                description=description,
+                key=path.name,
+                type="script" if path.suffix in {".R", ".py"} else "notebook",
+                revises=revises,
+            ).save()
+            logger.important(f"created Transform('{transform.uid}')")
+        if project is not None:
+            transform.projects.add(project_record)
+            logger.important(f"labeled with project: {project_record.name}")
         # latest run of this transform by user
         run = ln.Run.filter(transform=transform).order_by("-started_at").first()
         if run is not None and run.created_by.id != ln_setup.settings.user.id:
@@ -177,7 +231,7 @@ def save_from_filepath_cli(
         return_code = save_context_core(
             run=run,
             transform=transform,
-            filepath=filepath,
+            filepath=path,
             from_cli=True,
         )
         return return_code

{lamin_cli-1.3.0.dist-info → lamin_cli-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamin_cli
-Version: 1.3.0
+Version: 1.4.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

lamin_cli-1.4.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+lamin_cli/__init__.py,sha256=YeP3WxwX7_OcORoNOBGWEDB-_0yt4bX9J3ib92jScow,40
+lamin_cli/__main__.py,sha256=1GdJbaTQQmy_xPu-6QWglewBVkXdlGe37PbzSWPKCBM,12813
+lamin_cli/_cache.py,sha256=oplwE8AcS_9PYptQUZxff2qTIdNFS81clGPkJNWk098,800
+lamin_cli/_load.py,sha256=ow9IPQog_mPvjoGj1Bn08l4AG1YyRNnEyBpQvQkfLZs,8182
+lamin_cli/_migration.py,sha256=xQi6mwnpBzY5wcv1-TJhveD7a3XJIlpiYx6Z3AJ1NF0,1063
+lamin_cli/_save.py,sha256=LvxEIXKgUPvpwhjgRyxi1MTDRcL6rlAMbMQLfPtfDhs,9047
+lamin_cli/_settings.py,sha256=O2tecCf5EIZu98ima4DTJujo4KuywckOLgw8c-Ke3dY,1142
+lamin_cli/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lamin_cli/compute/modal.py,sha256=QnR7GyyvWWWkLnou95HxS9xxSQfw1k-SiefM_qRVnU0,6010
+lamin_cli-1.4.0.dist-info/entry_points.txt,sha256=Qms85i9cZPlu-U7RnVZhFsF7vJ9gaLZUFkCjcGcXTpg,49
+lamin_cli-1.4.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamin_cli-1.4.0.dist-info/WHEEL,sha256=ssQ84EZ5gH1pCOujd3iW7HClo_O_aDaClUbX4B8bjKY,100
+lamin_cli-1.4.0.dist-info/METADATA,sha256=GZKhv9Y-CgSgG32doVUMA2a42IZtjKyo0F-8YwgptCQ,337
+lamin_cli-1.4.0.dist-info/RECORD,,

lamin_cli-1.3.0.dist-info/RECORD

@@ -1,14 +0,0 @@
-lamin_cli/__init__.py,sha256=M6lfPWq9Iw7_uUcLoHjPp1SVuvhsIU0NkQQDt-EYpIc,40
-lamin_cli/__main__.py,sha256=2CZdjlQK0nHrMyDpNvKn7iQ34Owlbwhl4mmavc5yXgc,12392
-lamin_cli/_cache.py,sha256=oplwE8AcS_9PYptQUZxff2qTIdNFS81clGPkJNWk098,800
-lamin_cli/_load.py,sha256=lMhV9AMkybjvj4VChJE_v7IMy6qGqisFlo40BJUibsA,8087
-lamin_cli/_migration.py,sha256=xQi6mwnpBzY5wcv1-TJhveD7a3XJIlpiYx6Z3AJ1NF0,1063
-lamin_cli/_save.py,sha256=JQQxby2PP6SfUb1FZhnwNnp0fkBqTj-UnbGql4oPt8g,6948
-lamin_cli/_settings.py,sha256=O2tecCf5EIZu98ima4DTJujo4KuywckOLgw8c-Ke3dY,1142
-lamin_cli/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lamin_cli/compute/modal.py,sha256=QnR7GyyvWWWkLnou95HxS9xxSQfw1k-SiefM_qRVnU0,6010
-lamin_cli-1.3.0.dist-info/entry_points.txt,sha256=Qms85i9cZPlu-U7RnVZhFsF7vJ9gaLZUFkCjcGcXTpg,49
-lamin_cli-1.3.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamin_cli-1.3.0.dist-info/WHEEL,sha256=ssQ84EZ5gH1pCOujd3iW7HClo_O_aDaClUbX4B8bjKY,100
-lamin_cli-1.3.0.dist-info/METADATA,sha256=m6_U_cQGfZ-2HjU-u_1frMM-Q6ATjQCObBkiebqNzMs,337
-lamin_cli-1.3.0.dist-info/RECORD,,