lamindb 0.76.8__py3-none-any.whl → 0.76.10__py3-none-any.whl

lamindb/core/_settings.py

@@ -1,187 +1,188 @@
-from __future__ import annotations
-
-import os
-from typing import TYPE_CHECKING, Literal, Mapping
-
-import lamindb_setup as ln_setup
-from lamin_utils import logger
-from lamindb_setup._set_managed_storage import set_managed_storage
-from lamindb_setup.core._settings import settings as setup_settings
-from lamindb_setup.core._settings_instance import sanitize_git_repo_url
-
-from .subsettings._creation_settings import CreationSettings, creation_settings
-from .subsettings._transform_settings import TransformSettings, transform_settings
-
-if TYPE_CHECKING:
-    from pathlib import Path
-
-    from lamindb_setup.core._settings_storage import StorageSettings
-    from upath import UPath
-
-VERBOSITY_TO_INT = {
-    "error": 0,  # 40
-    "warning": 1,  # 30
-    "success": 2,  # 25
-    "info": 3,  # 20
-    "hint": 4,  # 15
-    "debug": 5,  # 10
-}
-VERBOSITY_TO_STR: dict[int, str] = dict(
-    [reversed(i) for i in VERBOSITY_TO_INT.items()]  # type: ignore
-)
-
-
-class Settings:
-    """Settings.
-
-    Use ``lamindb.settings`` instead of instantiating this class yourself.
-    """
-
-    def __init__(self, git_repo: str | None):
-        self._verbosity_int: int = 1  # warning-level logging
-        logger.set_verbosity(self._verbosity_int)
-        self._sync_git_repo: str | None = git_repo
-
-    @property
-    def creation(self) -> CreationSettings:
-        """Record creation settings.
-
-        For example, `ln.settings.creation.search_names = False` will disable
-        searching for records with similar names during creation.
-        """
-        return creation_settings
-
-    track_run_inputs: bool = True
-    """Track files as input upon `.load()`, `.cache()` and `.open()`.
-
-    Requires a global run context with :func:`~lamindb.core.Context.track` was created!
-
-    FAQ: :doc:`/faq/track-run-inputs`
-    """
-    __using_key: str | None = None
-    _using_storage: str | None = None
-
-    @property
-    def _using_key(self) -> str | None:
-        """Key for Django database settings."""
-        return self.__using_key
-
-    @_using_key.setter
-    def _using_key(self, value: str | None):
-        ln_setup.settings._using_key = value
-        self.__using_key = value
-
-    @property
-    def _storage_settings(self) -> ln_setup.core.StorageSettings:
-        if self._using_storage is None:
-            storage_settings = ln_setup.settings.storage
-        else:
-            storage_settings = ln_setup.core.StorageSettings(root=self._using_storage)
-        return storage_settings
-
-    @property
-    def transform(self) -> TransformSettings:
-        """Transform settings.
-
-        Is deprecated since version 0.76.1.
-        """
-        # enable warning soon
-        # logger.warning("Transform settings are deprecated, please instead set `ln.context.uid`")
-        return transform_settings
-
-    @property
-    def sync_git_repo(self) -> str | None:
-        """Sync transforms with scripts in git repository.
-
-        Provide the full git repo URL.
-        """
-        return self._sync_git_repo
-
-    @sync_git_repo.setter
-    def sync_git_repo(self, value) -> None:
-        """Sync transforms with scripts in git repository.
-
-        For example: `ln.sync_git_repo = https://github.com/laminlabs/redun-lamin`
-        """
-        self._sync_git_repo = sanitize_git_repo_url(value)
-        if not self._sync_git_repo.startswith("https://"):  # pragma: nocover
-            raise ValueError("git repository URL must start with 'https://'.")
-
-    @property
-    def storage(self) -> StorageSettings:
-        """Default storage location.
-
-        Examples:
-
-        >>> ln.settings.storage
-        StorageSettings(root='s3://my-bucket', uid='j7MaPxtLxPeE')
-
-        >>> ln.settings.storage.root
-        UPath('s3://my-bucket')
-
-        You can switch the default storage location to another managed storage
-        location by passing a string:
-
-        >>> ln.settings.storage = "s3://some-bucket"
-
-        You can also pass additional fsspec kwargs via:
-
-        >>> kwargs = dict(
-        >>>     profile="some_profile", # fsspec arg
-        >>>     cache_regions=True # fsspec arg for s3
-        >>> )
-        >>> ln.settings.storage = "s3://some-bucket", kwargs
-        """
-        return self._storage_settings
-
-    @storage.setter
-    def storage(self, path_kwargs: str | Path | UPath | tuple[str | UPath, Mapping]):
-        if isinstance(path_kwargs, tuple):
-            path, kwargs = path_kwargs
-        else:
-            path, kwargs = path_kwargs, {}
-        set_managed_storage(path, **kwargs)
-
-    @property
-    def storage_local(self) -> StorageSettings:
-        """An additional local default storage (a path to its root).
-
-        Is only available if :attr:`~lamindb.setup.core.InstanceSettings.keep_artifacts_local` is enabled.
-
-        Guide: :doc:`faq/keep-artifacts-local`
-        """
-        return ln_setup.settings.instance.storage_local
-
-    @storage_local.setter
-    def storage_local(self, local_root: Path):
-        ln_setup.settings.instance.storage_local = local_root
-
-    @property
-    def verbosity(self) -> str:
-        """Logger verbosity (default `'warning'`).
-
-        - `'error'`: ❌ only show error messages
-        - `'warning'`: ❗ also show warning messages
-        - `'success'`: ✅ also show success and save messages
-        - `'info'`: 💡 also show info messages
-        - `'hint'`: 💡 also show hint messages
-        - `'debug'`: 🐛 also show detailed debug messages
-        """
-        return VERBOSITY_TO_STR[self._verbosity_int]
-
-    @verbosity.setter
-    def verbosity(self, verbosity: str | int):
-        if isinstance(verbosity, str):
-            verbosity_int = VERBOSITY_TO_INT[verbosity]
-        else:
-            verbosity_int = verbosity
-        self._verbosity_int = verbosity_int
-        logger.set_verbosity(verbosity_int)
-
-
-if os.environ.get("LAMINDB_MULTI_INSTANCE") == "true":
-    git_repo = None
-else:
-    git_repo = setup_settings.instance.git_repo
-
-settings = Settings(git_repo=git_repo)
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING, Literal
+
+import lamindb_setup as ln_setup
+from lamin_utils import logger
+from lamindb_setup._set_managed_storage import set_managed_storage
+from lamindb_setup.core._settings import settings as setup_settings
+from lamindb_setup.core._settings_instance import sanitize_git_repo_url
+
+from .subsettings._creation_settings import CreationSettings, creation_settings
+from .subsettings._transform_settings import TransformSettings, transform_settings
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from pathlib import Path
+
+    from lamindb_setup.core._settings_storage import StorageSettings
+    from upath import UPath
+
+VERBOSITY_TO_INT = {
+    "error": 0,  # 40
+    "warning": 1,  # 30
+    "success": 2,  # 25
+    "info": 3,  # 20
+    "hint": 4,  # 15
+    "debug": 5,  # 10
+}
+VERBOSITY_TO_STR: dict[int, str] = dict(
+    [reversed(i) for i in VERBOSITY_TO_INT.items()]  # type: ignore
+)
+
+
+class Settings:
+    """Settings.
+
+    Use ``lamindb.settings`` instead of instantiating this class yourself.
+    """
+
+    def __init__(self, git_repo: str | None):
+        self._verbosity_int: int = 1  # warning-level logging
+        logger.set_verbosity(self._verbosity_int)
+        self._sync_git_repo: str | None = git_repo
+
+    @property
+    def creation(self) -> CreationSettings:
+        """Record creation settings.
+
+        For example, `ln.settings.creation.search_names = False` will disable
+        searching for records with similar names during creation.
+        """
+        return creation_settings
+
+    track_run_inputs: bool = True
+    """Track files as input upon `.load()`, `.cache()` and `.open()`.
+
+    Requires a global run context with :func:`~lamindb.core.Context.track` was created!
+
+    FAQ: :doc:`/faq/track-run-inputs`
+    """
+    __using_key: str | None = None
+    _using_storage: str | None = None
+
+    @property
+    def _using_key(self) -> str | None:
+        """Key for Django database settings."""
+        return self.__using_key
+
+    @_using_key.setter
+    def _using_key(self, value: str | None):
+        ln_setup.settings._using_key = value
+        self.__using_key = value
+
+    @property
+    def _storage_settings(self) -> ln_setup.core.StorageSettings:
+        if self._using_storage is None:
+            storage_settings = ln_setup.settings.storage
+        else:
+            storage_settings = ln_setup.core.StorageSettings(root=self._using_storage)
+        return storage_settings
+
+    @property
+    def transform(self) -> TransformSettings:
+        """Transform settings.
+
+        Is deprecated since version 0.76.1.
+        """
+        # enable warning soon
+        # logger.warning("Transform settings are deprecated, please instead set `ln.context.uid`")
+        return transform_settings
+
+    @property
+    def sync_git_repo(self) -> str | None:
+        """Sync transforms with scripts in git repository.
+
+        Provide the full git repo URL.
+        """
+        return self._sync_git_repo
+
+    @sync_git_repo.setter
+    def sync_git_repo(self, value) -> None:
+        """Sync transforms with scripts in git repository.
+
+        For example: `ln.sync_git_repo = https://github.com/laminlabs/redun-lamin`
+        """
+        self._sync_git_repo = sanitize_git_repo_url(value)
+        if not self._sync_git_repo.startswith("https://"):  # pragma: nocover
+            raise ValueError("git repository URL must start with 'https://'.")
+
+    @property
+    def storage(self) -> StorageSettings:
+        """Default storage location.
+
+        Examples:
+
+        >>> ln.settings.storage
+        StorageSettings(root='s3://my-bucket', uid='j7MaPxtLxPeE')
+
+        >>> ln.settings.storage.root
+        UPath('s3://my-bucket')
+
+        You can switch the default storage location to another managed storage
+        location by passing a string:
+
+        >>> ln.settings.storage = "s3://some-bucket"
+
+        You can also pass additional fsspec kwargs via:
+
+        >>> kwargs = dict(
+        >>>     profile="some_profile", # fsspec arg
+        >>>     cache_regions=True # fsspec arg for s3
+        >>> )
+        >>> ln.settings.storage = "s3://some-bucket", kwargs
+        """
+        return self._storage_settings
+
+    @storage.setter
+    def storage(self, path_kwargs: str | Path | UPath | tuple[str | UPath, Mapping]):
+        if isinstance(path_kwargs, tuple):
+            path, kwargs = path_kwargs
+        else:
+            path, kwargs = path_kwargs, {}
+        set_managed_storage(path, **kwargs)
+
+    @property
+    def storage_local(self) -> StorageSettings:
+        """An additional local default storage (a path to its root).
+
+        Is only available if :attr:`~lamindb.setup.core.InstanceSettings.keep_artifacts_local` is enabled.
+
+        Guide: :doc:`faq/keep-artifacts-local`
+        """
+        return ln_setup.settings.instance.storage_local
+
+    @storage_local.setter
+    def storage_local(self, local_root: Path):
+        ln_setup.settings.instance.storage_local = local_root
+
+    @property
+    def verbosity(self) -> str:
+        """Logger verbosity (default `'warning'`).
+
+        - `'error'`: ❌ only show error messages
+        - `'warning'`: ❗ also show warning messages
+        - `'success'`: ✅ also show success and save messages
+        - `'info'`: 💡 also show info messages
+        - `'hint'`: 💡 also show hint messages
+        - `'debug'`: 🐛 also show detailed debug messages
+        """
+        return VERBOSITY_TO_STR[self._verbosity_int]
+
+    @verbosity.setter
+    def verbosity(self, verbosity: str | int):
+        if isinstance(verbosity, str):
+            verbosity_int = VERBOSITY_TO_INT[verbosity]
+        else:
+            verbosity_int = verbosity
+        self._verbosity_int = verbosity_int
+        logger.set_verbosity(verbosity_int)
+
+
+if os.environ.get("LAMINDB_MULTI_INSTANCE") == "true":
+    git_repo = None
+else:
+    git_repo = setup_settings.instance.git_repo
+
+settings = Settings(git_repo=git_repo)

lamindb/core/_sync_git.py

@@ -1,138 +1,138 @@
-from __future__ import annotations
-
-import subprocess
-from pathlib import Path
-
-from lamin_utils import logger
-from lamindb_setup import settings as setup_settings
-from lamindb_setup.core.hashing import hash_code
-
-from ._settings import sanitize_git_repo_url, settings
-
-
-class BlobHashNotFound(SystemExit):
-    pass
-
-
-def get_git_repo_from_remote() -> Path:
-    repo_url = settings.sync_git_repo
-    repo_dir = setup_settings.storage.cache_dir / repo_url.split("/")[-1]
-    if repo_dir.exists():
-        logger.warning(f"git repo {repo_dir} already exists locally")
-        return repo_dir
-    logger.important(
-        f"running outside of synched git repo, cloning {repo_url} into {repo_dir}"
-    )
-    result = subprocess.run(
-        ["git", "clone", "--depth", "10", f"{repo_url}.git"],
-        capture_output=True,
-        cwd=setup_settings.storage.cache_dir,
-    )
-    if result.returncode != 0 or not repo_dir.exists():
-        raise RuntimeError(result.stderr.decode())
-    return repo_dir
-
-
-def check_local_git_repo() -> bool:
-    result = subprocess.run(
-        ["git", "config", "--get remote.origin.url"],
-        capture_output=True,
-    )
-    result_str = result.stdout.decode().strip()
-    if result_str == "":
-        # running-not-in-a-git-repo
-        return False
-    else:
-        remote_url = sanitize_git_repo_url(result_str)
-        if remote_url == settings.sync_git_repo:
-            # running-in-correct-git-repo
-            return True
-        else:
-            # running-outside-of-correct-git-repo
-            return False
-
-
-def get_git_commit_hash(blob_hash: str, repo_dir: Path | None = None) -> str | None:
-    command = ["git", "log", f"--find-object={blob_hash}", "--pretty=format:%H"]
-    result = subprocess.run(
-        command,
-        capture_output=True,
-        cwd=repo_dir,
-    )
-    # we just care to find one commit
-    # hence, we split by new line ("\n") and use the first one
-    commit_hash = result.stdout.decode().split("\n")[0]
-    if commit_hash == "" or result.returncode == 1:
-        return None
-    else:
-        assert (  # noqa: S101
-            len(commit_hash) == 40
-        ), f"commit hash |{commit_hash}| is not 40 characters long"
-        return commit_hash
-
-
-def get_filepath_within_git_repo(
-    commit_hash: str, blob_hash: str, repo_dir: Path | None
-) -> str:
-    # repo_dir might not point to the root of the
-    # the git repository because git log --find-object works
-    # from anywhere in the repo, hence, let's get the root
-    repo_root = (
-        subprocess.run(
-            ["git", "rev-parse", "--show-toplevel"],
-            capture_output=True,
-            cwd=repo_dir,
-        )
-        .stdout.decode()
-        .strip()
-    )
-    # Run the git commands separately to circumvent spawning a shell
-    git_command = ["git", "ls-tree", "-r", commit_hash]
-    git_process = subprocess.Popen(
-        git_command,
-        stdout=subprocess.PIPE,
-        cwd=repo_root,
-    )
-
-    grep_command = ["grep", "-E", blob_hash]
-    result = subprocess.run(
-        grep_command,
-        stdin=git_process.stdout,
-        capture_output=True,
-        cwd=repo_root,
-    )
-
-    # Close the stdout to allow git_process to receive a SIGPIPE if grep_command exits
-    git_process.stdout.close()
-    git_process.wait()
-
-    command = " ".join(git_command) + " | " + " ".join(grep_command)
-    if result.returncode != 0 and result.stderr.decode() != "":
-        raise RuntimeError(f"{command}\n{result.stderr.decode()}")
-    if len(result.stdout.decode()) == 0:
-        raise RuntimeError(
-            f"Could not find path in git repo {settings.sync_git_repo} running:\n{command}"
-            f"\nin local clone: {repo_root}"
-        )
-    filepath = result.stdout.decode().split()[-1]
-    return filepath
-
-
-def get_transform_reference_from_git_repo(path: Path) -> str:
-    blob_hash = hash_code(path).hexdigest()
-    commit_hash = None
-    if check_local_git_repo():
-        repo_dir = None
-    else:
-        repo_dir = get_git_repo_from_remote()
-    commit_hash = get_git_commit_hash(blob_hash, repo_dir=repo_dir)
-    if commit_hash is None:
-        if repo_dir is None:
-            repo_dir = Path.cwd()
-        raise BlobHashNotFound(
-            f"❌ Did not find blob hash {blob_hash} in git repo ({settings.sync_git_repo}) {repo_dir}\n"
-            f"Did you commit the script? -> {path}"
-        )
-    gitpath = get_filepath_within_git_repo(commit_hash, blob_hash, repo_dir)
-    reference = f"{settings.sync_git_repo}/blob/{commit_hash}/{gitpath}"
-    return reference
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from lamin_utils import logger
+from lamindb_setup import settings as setup_settings
+from lamindb_setup.core.hashing import hash_code
+
+from ._settings import sanitize_git_repo_url, settings
+
+
+class BlobHashNotFound(SystemExit):
+    pass
+
+
+def get_git_repo_from_remote() -> Path:
+    repo_url = settings.sync_git_repo
+    repo_dir = setup_settings.storage.cache_dir / repo_url.split("/")[-1]
+    if repo_dir.exists():
+        logger.warning(f"git repo {repo_dir} already exists locally")
+        return repo_dir
+    logger.important(
+        f"running outside of synched git repo, cloning {repo_url} into {repo_dir}"
+    )
+    result = subprocess.run(
+        ["git", "clone", "--depth", "10", f"{repo_url}.git"],
+        capture_output=True,
+        cwd=setup_settings.storage.cache_dir,
+    )
+    if result.returncode != 0 or not repo_dir.exists():
+        raise RuntimeError(result.stderr.decode())
+    return repo_dir
+
+
+def check_local_git_repo() -> bool:
+    result = subprocess.run(
+        ["git", "config", "--get remote.origin.url"],
+        capture_output=True,
+    )
+    result_str = result.stdout.decode().strip()
+    if result_str == "":
+        # running-not-in-a-git-repo
+        return False
+    else:
+        remote_url = sanitize_git_repo_url(result_str)
+        if remote_url == settings.sync_git_repo:
+            # running-in-correct-git-repo
+            return True
+        else:
+            # running-outside-of-correct-git-repo
+            return False
+
+
+def get_git_commit_hash(blob_hash: str, repo_dir: Path | None = None) -> str | None:
+    command = ["git", "log", f"--find-object={blob_hash}", "--pretty=format:%H"]
+    result = subprocess.run(
+        command,
+        capture_output=True,
+        cwd=repo_dir,
+    )
+    # we just care to find one commit
+    # hence, we split by new line ("\n") and use the first one
+    commit_hash = result.stdout.decode().split("\n")[0]
+    if commit_hash == "" or result.returncode == 1:
+        return None
+    else:
+        assert (  # noqa: S101
+            len(commit_hash) == 40
+        ), f"commit hash |{commit_hash}| is not 40 characters long"
+        return commit_hash
+
+
+def get_filepath_within_git_repo(
+    commit_hash: str, blob_hash: str, repo_dir: Path | None
+) -> str:
+    # repo_dir might not point to the root of the
+    # the git repository because git log --find-object works
+    # from anywhere in the repo, hence, let's get the root
+    repo_root = (
+        subprocess.run(
+            ["git", "rev-parse", "--show-toplevel"],
+            capture_output=True,
+            cwd=repo_dir,
+        )
+        .stdout.decode()
+        .strip()
+    )
+    # Run the git commands separately to circumvent spawning a shell
+    git_command = ["git", "ls-tree", "-r", commit_hash]
+    git_process = subprocess.Popen(
+        git_command,
+        stdout=subprocess.PIPE,
+        cwd=repo_root,
+    )
+
+    grep_command = ["grep", "-E", blob_hash]
+    result = subprocess.run(
+        grep_command,
+        stdin=git_process.stdout,
+        capture_output=True,
+        cwd=repo_root,
+    )
+
+    # Close the stdout to allow git_process to receive a SIGPIPE if grep_command exits
+    git_process.stdout.close()
+    git_process.wait()
+
+    command = " ".join(git_command) + " | " + " ".join(grep_command)
+    if result.returncode != 0 and result.stderr.decode() != "":
+        raise RuntimeError(f"{command}\n{result.stderr.decode()}")
+    if len(result.stdout.decode()) == 0:
+        raise RuntimeError(
+            f"Could not find path in git repo {settings.sync_git_repo} running:\n{command}"
+            f"\nin local clone: {repo_root}"
+        )
+    filepath = result.stdout.decode().split()[-1]
+    return filepath
+
+
+def get_transform_reference_from_git_repo(path: Path) -> str:
+    blob_hash = hash_code(path).hexdigest()
+    commit_hash = None
+    if check_local_git_repo():
+        repo_dir = None
+    else:
+        repo_dir = get_git_repo_from_remote()
+    commit_hash = get_git_commit_hash(blob_hash, repo_dir=repo_dir)
+    if commit_hash is None:
+        if repo_dir is None:
+            repo_dir = Path.cwd()
+        raise BlobHashNotFound(
+            f"❌ Did not find blob hash {blob_hash} in git repo ({settings.sync_git_repo}) {repo_dir}\n"
+            f"Did you commit the script? -> {path}"
+        )
+    gitpath = get_filepath_within_git_repo(commit_hash, blob_hash, repo_dir)
+    reference = f"{settings.sync_git_repo}/blob/{commit_hash}/{gitpath}"
+    return reference