lamin_cli 1.10.0__tar.gz → 1.11.0__tar.gz

{lamin_cli-1.10.0 → lamin_cli-1.11.0}/PKG-INFO

@@ -1,10 +1,9 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: lamin_cli
-Version: 1.10.0
+Version: 1.11.0
 Summary: Lamin CLI.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: rich-click>=1.7
 Project-URL: Home, https://github.com/laminlabs/lamin-cli
 

{lamin_cli-1.10.0 → lamin_cli-1.11.0}/lamin_cli/__init__.py

@@ -6,7 +6,7 @@ The interface is defined in `__main__.py`.
 The root API here is used by LaminR to replicate the CLI functionality.
 """
 
-__version__ = "1.10.0"
+__version__ = "1.11.0"
 
 from lamindb_setup import disconnect, logout
 from lamindb_setup._connect_instance import _connect_cli as connect

{lamin_cli-1.10.0 → lamin_cli-1.11.0}/lamin_cli/__main__.py

@@ -42,6 +42,10 @@ COMMAND_GROUPS = {
             "name": "Load, save, create & delete data",
             "commands": ["load", "save", "create", "delete"],
         },
+        {
+            "name": "Tracking within shell scripts",
+            "commands": ["track", "finish"],
+        },
         {
             "name": "Describe, annotate & list data",
             "commands": ["describe", "annotate", "list"],
@@ -436,6 +440,41 @@ def save(path: str, key: str, description: str, stem_uid: str, project: str, spa
     if save_(path=path, key=key, description=description, stem_uid=stem_uid, project=project, space=space, branch=branch, registry=registry) is not None:
         sys.exit(1)
 
+@main.command()
+def track():
+    """Start tracking a run of a shell script.
+
+    This command works like {func}`~lamindb.track()` in a Python session. Here is an example script:
+
+    ```
+    # my_script.sh
+    set -e         # exit on error
+    lamin track    # initiate a tracked shell script run
+    lamin load --key raw/file1.txt
+    # do something
+    lamin save processed_file1.txt --key processed/file1.txt
+    lamin finish   # mark the shell script run as finished
+    ```
+
+    If you run that script, it will track the run of the script, and save the input and output artifacts:
+
+    ```
+    sh my_script.sh
+    ```
+    """
+    from lamin_cli._context import track as track_
+    return track_()
+
+
+@main.command()
+def finish():
+    """Finish the current tracked run of a shell script.
+
+    This command works like {func}`~lamindb.finish()` in a Python session.
+    """
+    from lamin_cli._context import finish as finish_
+    return finish_()
+
 
 @main.command()
 @click.argument("entity", type=str, default=None, required=False)

lamin_cli-1.11.0/lamin_cli/_context.py

@@ -0,0 +1,76 @@
+import os
+import sys
+from pathlib import Path
+
+import click
+from lamin_utils import logger
+from lamindb_setup.core._settings_store import settings_dir
+
+
+def get_current_run_file() -> Path:
+    """Get the path to the file storing the current run UID."""
+    return settings_dir / "current_shell_run.txt"
+
+
+def is_interactive_shell() -> bool:
+    """Check if running in an interactive terminal."""
+    return sys.stdin.isatty() and sys.stdout.isatty() and os.isatty(0)
+
+
+def get_script_filename() -> Path:
+    """Try to get the filename of the calling shell script."""
+    import psutil
+
+    parent = psutil.Process(os.getppid())
+    cmdline = parent.cmdline()
+
+    # For shells like bash, sh, zsh
+    if parent.name() in ["bash", "sh", "zsh", "dash"]:
+        # cmdline is typically: ['/bin/bash', 'script.sh', ...]
+        if len(cmdline) > 1 and not cmdline[1].startswith("-"):
+            return Path(cmdline[1])
+    raise click.ClickException(
+        "Cannot determine script filename. Please run in an interactive shell."
+    )
+
+
+def track():
+    import lamindb as ln
+
+    if not ln.setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+    path = get_script_filename()
+    source_code = path.read_text()
+    transform = ln.Transform(
+        key=path.name, source_code=source_code, type="script"
+    ).save()
+    run = ln.Run(transform=transform).save()
+    current_run_file = get_current_run_file()
+    current_run_file.parent.mkdir(parents=True, exist_ok=True)
+    current_run_file.write_text(run.uid)
+    logger.important(f"started tracking shell run: {run.uid}")
+
+
+def finish():
+    from datetime import datetime, timezone
+
+    import lamindb as ln
+
+    if not ln.setup.settings._instance_exists:
+        raise click.ClickException(
+            "Not connected to an instance. Please run: lamin connect account/name"
+        )
+
+    current_run_file = get_current_run_file()
+    if not current_run_file.exists():
+        raise click.ClickException(
+            "No active run to finish. Please run `lamin track` first."
+        )
+    run = ln.Run.get(uid=current_run_file.read_text().strip())
+    run._status_code = 0
+    run.finished_at = datetime.now(timezone.utc)
+    run.save()
+    current_run_file.unlink()
+    logger.important(f"finished tracking shell run: {run.uid}")

{lamin_cli-1.10.0 → lamin_cli-1.11.0}/lamin_cli/_io.py

@@ -30,7 +30,7 @@ def io():
 @click.option("--track/--no-track", is_flag=True, help="Whether to track snapshot generation.", default=True)
 # fmt: on
 def snapshot(upload: bool, track: bool) -> None:
-    """Create and optionally upload a SQLite snapshot of the current instance."""
+    """Create a SQLite snapshot of the connected instance."""
     if not ln_setup.settings._instance_exists:
         raise click.ClickException(
             "Not connected to an instance. Please run: lamin connect account/name"
@@ -86,8 +86,8 @@ def snapshot(upload: bool, track: bool) -> None:
                     for table, (orig, clone) in mismatches.items()]
                 )
                 raise click.ClickException(error_msg)
-            except json.JSONDecodeError:
-                error_msg = f"Clone verification failed:\n{result.stderr}"
+            except (json.JSONDecodeError, AttributeError, ValueError, TypeError):
+                raise click.ClickException(f"Clone verification failed:\n{result.stderr}") from None
 
 
         ln_setup.connect(f"{instance_owner}/{instance_name}", use_root_db_user=True)

{lamin_cli-1.10.0 → lamin_cli-1.11.0}/lamin_cli/_load.py

@@ -6,6 +6,7 @@ from pathlib import Path
 
 from lamin_utils import logger
 
+from ._context import get_current_run_file
 from ._save import infer_registry_from_path, parse_title_r_notebook
 from .urls import decompose_url
 
@@ -48,6 +49,10 @@ def load(
     ln_setup.connect(instance)
     import lamindb as ln
 
+    current_run = None
+    if get_current_run_file().exists():
+        current_run = ln.Run.get(uid=get_current_run_file().read_text().strip())
+
     def script_to_notebook(
         transform: ln.Transform, notebook_path: Path, bump_revision: bool = False
     ) -> None:
@@ -182,7 +187,7 @@ def load(
                 entities = entities.order_by("-created_at")
 
             entity_obj = entities.first()
-            cache_path = entity_obj.cache()
+            cache_path = entity_obj.cache(is_run_input=current_run)
 
             # collection gives us a list of paths
             if isinstance(cache_path, list):

{lamin_cli-1.10.0 → lamin_cli-1.11.0}/lamin_cli/_save.py

@@ -9,17 +9,23 @@ import lamindb_setup as ln_setup
 from lamin_utils import logger
 from lamindb_setup.core.hashing import hash_file
 
+from lamin_cli._context import get_current_run_file
+
 
 def infer_registry_from_path(path: Path | str) -> str:
     suffixes_transform = {
         "py": {".py", ".ipynb"},
         "R": {".R", ".qmd", ".Rmd"},
+        "sh": {".sh"},
     }
     if isinstance(path, str):
         path = Path(path)
     registry = (
         "transform"
-        if path.suffix in suffixes_transform["py"].union(suffixes_transform["R"])
+        if path.suffix
+        in suffixes_transform["py"]
+        .union(suffixes_transform["R"])
+        .union(suffixes_transform["sh"])
         else "artifact"
     )
     return registry
@@ -42,9 +48,11 @@ def parse_uid_from_code(content: str, suffix: str) -> str | None:
             r'track\(\s*(?:transform\s*=\s*)?([\'"])([a-zA-Z0-9]{12,16})\1'
         )
         uid_pattern = None
+    elif suffix == ".sh":
+        return None
     else:
         raise SystemExit(
-            "Only .py, .ipynb, .R, .qmd, .Rmd files are supported for saving"
+            "Only .py, .ipynb, .R, .qmd, .Rmd, .sh files are supported for saving"
             " transforms."
         )
 
@@ -82,8 +90,13 @@ def save(
 ) -> str | None:
     import lamindb as ln
     from lamindb._finish import save_context_core
+    from lamindb_setup.core._settings_store import settings_dir
     from lamindb_setup.core.upath import LocalPathClasses, UPath, create_path
 
+    current_run = None
+    if get_current_run_file().exists():
+        current_run = ln.Run.get(uid=get_current_run_file().read_text().strip())
+
     # this allows to have the correct treatment of credentials in case of cloud paths
     ppath = create_path(path)
     # isinstance is needed to cast the type of path to UPath
@@ -149,6 +162,7 @@ def save(
             revises=revises,
             branch=branch_record,
             space=space_record,
+            run=current_run,
         ).save()
         logger.important(f"saved: {artifact}")
         logger.important(f"storage path: {artifact.path}")
@@ -219,7 +233,7 @@ def save(
                 if transform is None:
                     uid = f"{stem_uid}0000"
         else:
-            transform_hash, _ = hash_file(ppath)
+            _, transform_hash, _ = hash_file(ppath)
             transform = ln.Transform.filter(hash=transform_hash).first()
             if transform is not None and transform.hash is not None:
                 if transform.hash == transform_hash:
@@ -268,7 +282,7 @@ def save(
                 uid=uid,
                 description=description,
                 key=key,
-                type="script" if ppath.suffix in {".R", ".py"} else "notebook",
+                type="script" if ppath.suffix in {".R", ".py", ".sh"} else "notebook",
                 revises=revises,
             )
             if space is not None:

lamin_cli-1.11.0/tests/core/test_save_shell_script.py

@@ -0,0 +1,38 @@
+import os
+import subprocess
+from pathlib import Path
+
+import lamindb as ln
+
+scripts_dir = Path(__file__).parent.parent.resolve() / "scripts"
+
+
+def test_save_shell_script():
+    """Test that saving a shell script creates a transform with source_code and type='script'."""
+    env = os.environ.copy()
+    env["LAMIN_TESTING"] = "true"
+
+    script_path = scripts_dir / "test-shell-script.sh"
+    script_content = "#!/bin/bash\necho 'test script'\n"
+    script_path.write_text(script_content)
+
+    result = subprocess.run(
+        f"lamin save {script_path}",
+        shell=True,
+        env=env,
+        capture_output=True,
+    )
+    assert result.returncode == 0
+    assert "created Transform" in result.stdout.decode()
+
+    transform = ln.Transform.get(key=script_path.name)
+
+    assert transform.source_code is not None, (
+        "Transform source_code should be populated"
+    )
+    assert len(transform.source_code) > 0, "Transform source_code should not be empty"
+    assert transform.type == "script", (
+        f"Transform type should be 'script', got '{transform.type}'"
+    )
+
+    script_path.unlink()

lamin_cli-1.11.0/tests/core/test_track.py

@@ -0,0 +1,65 @@
+import os
+import subprocess
+from pathlib import Path
+
+import lamindb as ln
+
+scripts_dir = Path(__file__).parent.parent.resolve() / "scripts"
+
+
+def test_track_lineage_via_cli():
+    """Test that lineage tracking works correctly via CLI commands."""
+    env = os.environ.copy()
+    env["LAMIN_TESTING"] = "true"
+
+    script_path = scripts_dir / "track-lineage.sh"
+
+    # Make the script executable
+    script_path.chmod(0o755)
+
+    # Run the shell script
+    result = subprocess.run(
+        ["sh", str(script_path)],
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+
+    # Check that the script ran successfully
+    if result.returncode != 0:
+        print(f"Script stdout: {result.stdout}")
+        print(f"Script stderr: {result.stderr}")
+    assert result.returncode == 0, (
+        f"Script failed:\nstdout: {result.stdout}\nstderr: {result.stderr}"
+    )
+
+    transform = ln.Transform.get(key="track-lineage.sh")
+    assert transform.type == "script"
+    run = transform.latest_run
+
+    input_artifacts = run.input_artifacts.all()
+    assert len(input_artifacts) == 1, (
+        f"Expected 1 input artifact, got {len(input_artifacts)}"
+    )
+    input_artifact = input_artifacts[0]
+    assert input_artifact.key == "test/input.txt", (
+        f"Expected input artifact key 'test/input.txt', got '{input_artifact.key}'"
+    )
+
+    output_artifacts = run.output_artifacts.all()
+    assert len(output_artifacts) == 1, (
+        f"Expected 1 output artifact, got {len(output_artifacts)}"
+    )
+    output_artifact = output_artifacts[0]
+    assert output_artifact.key == "test/output.txt", (
+        f"Expected output artifact key 'test/output.txt', got '{output_artifact.key}'"
+    )
+
+    assert run.finished_at is not None, "Run should be finished"
+    assert run._status_code == 0, "Run status code should be 0 (finished)"
+
+    # Clean up
+    input_artifact.delete(permanent=True)
+    output_artifact.delete(permanent=True)
+    run.delete(permanent=True)
+    transform.delete(permanent=True)

lamin_cli-1.11.0/tests/scripts/track-lineage.sh

@@ -0,0 +1,15 @@
+# prepare test data
+TEST_DIR=$(mktemp -d)
+trap "rm -rf $TEST_DIR" EXIT
+INPUT_FILE="$TEST_DIR/test_input.txt"
+OUTPUT_FILE="$TEST_DIR/test_output.txt"
+echo "test input data" > "$INPUT_FILE"
+lamin save "$INPUT_FILE" --key test/input.txt
+
+# actual script
+set -e  # exit on error
+lamin track
+lamin load --key test/input.txt
+echo "test output data" > "$OUTPUT_FILE"
+lamin save "$OUTPUT_FILE" --key test/output.txt
+lamin finish