lamindb 0.77.3__py3-none-any.whl → 1.0rc1__py3-none-any.whl

lamindb/core/_label_manager.py

@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
 
 from django.db import connections
 from lamin_utils import logger
-from lnschema_core.models import CanCurate, Feature
 from rich.table import Column, Table
 from rich.text import Text
 from rich.tree import Tree
@@ -19,6 +18,7 @@ from lamindb._record import (
     transfer_to_default_db,
 )
 from lamindb._save import save
+from lamindb.models import CanCurate, Feature
 
 from ._describe import (
     NAME_WIDTH,
@@ -29,14 +29,13 @@ from ._describe import (
 )
 from ._django import get_artifact_with_related, get_related_model
 from ._settings import settings
-from .schema import dict_related_model_to_related_name
+from .relations import dict_related_model_to_related_name
 
 if TYPE_CHECKING:
-    from lnschema_core.models import Artifact, Collection, Record
-
     from lamindb._query_set import QuerySet
+    from lamindb.models import Artifact, Collection, Record
 
-EXCLUDE_LABELS = {"feature_sets"}
+EXCLUDE_LABELS = {"_schemas_m2m"}
 
 
 def _get_labels(
@@ -107,7 +106,7 @@ def describe_labels(
         pad_edge=False,
     )
     for related_name, labels in labels_data.items():
-        if not labels or related_name == "feature_sets":
+        if not labels or related_name == "_schemas_m2m":
             continue
         if isinstance(labels, dict):  # postgres, labels are a dict[id, name]
             print_values = _format_values(labels.values(), n=10, quotes=False)
@@ -118,7 +117,7 @@ def describe_labels(
             )
         if print_values:
             related_model = get_related_model(self, related_name)
-            type_str = related_model.__get_name_with_schema__()
+            type_str = related_model.__get_name_with_module__()
             labels_table.add_row(
                 f".{related_name}", Text(type_str, style="dim"), print_values
             )
@@ -311,7 +310,7 @@ class LabelManager:
         """
         d = dict_related_model_to_related_name(self._host)
         registry = label.__class__
-        related_name = d.get(registry.__get_name_with_schema__())
+        related_name = d.get(registry.__get_name_with_module__())
         link_model = getattr(self._host, related_name).through
         link_records = link_model.filter(
             artifact_id=self._host.id, **{f"{registry.__name__.lower()}_id": label.id}

lamindb/core/_mapped_collection.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 from collections import Counter
 from functools import reduce
-from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 
 import numpy as np
@@ -86,9 +85,9 @@ class MappedCollection:
             retrieves ``.X``.
         obsm_keys: Keys from the ``.obsm`` slots.
         obs_keys: Keys from the ``.obs`` slots.
-        obs_filter: Select only observations with these values for the given obs column.
-            Should be a tuple with an obs column name as the first element
-            and filtering values (a string or a tuple of strings) as the second element.
+        obs_filter: Select only observations with these values for the given obs columns.
+            Should be a dictionary with obs column names as keys
+            and filtering values (a string or a tuple of strings) as values.
         join: `"inner"` or `"outer"` virtual joins. If ``None`` is passed,
             does not join.
         encode_labels: Encode labels into integers.
@@ -107,7 +106,7 @@ class MappedCollection:
         layers_keys: str | list[str] | None = None,
         obs_keys: str | list[str] | None = None,
         obsm_keys: str | list[str] | None = None,
-        obs_filter: tuple[str, str | tuple[str, ...]] | None = None,
+        obs_filter: dict[str, str | tuple[str, ...]] | None = None,
         join: Literal["inner", "outer"] | None = "inner",
         encode_labels: bool | list[str] = True,
         unknown_label: str | dict[str, str] | None = None,
@@ -121,11 +120,11 @@ class MappedCollection:
             )
 
         self.filtered = obs_filter is not None
-        if self.filtered and len(obs_filter) != 2:
-            raise ValueError(
-                "obs_filter should be a tuple with obs column name "
-                "as the first element and filtering values as the second element"
+        if self.filtered and not isinstance(obs_filter, dict):
+            logger.warning(
+                "Passing a tuple to `obs_filter` is deprecated, use a dictionary"
             )
+            obs_filter = {obs_filter[0]: obs_filter[1]}
 
         if layers_keys is None:
             self.layers_keys = ["X"]
@@ -183,12 +182,16 @@ class MappedCollection:
                 store_path = self.path_list[i]
                 self._check_csc_raise_error(X, "X", store_path)
                 if self.filtered:
-                    obs_filter_key, obs_filter_values = obs_filter
-                    indices_storage = np.where(
-                        np.isin(
+                    indices_storage_mask = None
+                    for obs_filter_key, obs_filter_values in obs_filter.items():
+                        obs_filter_mask = np.isin(
                             self._get_labels(store, obs_filter_key), obs_filter_values
                         )
-                    )[0]
+                        if indices_storage_mask is None:
+                            indices_storage_mask = obs_filter_mask
+                        else:
+                            indices_storage_mask &= obs_filter_mask
+                    indices_storage = np.where(indices_storage_mask)[0]
                     n_obs_storage = len(indices_storage)
                 else:
                     if isinstance(X, ArrayTypes):  # type: ignore
@@ -348,7 +351,7 @@ class MappedCollection:
 
     @property
     def original_shapes(self) -> list[tuple[int, int]]:
-        """Shapes of the underlying AnnData objects."""
+        """Shapes of the underlying AnnData objects (with `obs_filter` applied)."""
         if self.n_vars_list is None:
             n_vars_list = [None] * len(self.n_obs_list)
         else:

lamindb/core/_settings.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import os
-from typing import TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
 from lamin_utils import logger
@@ -10,7 +10,6 @@ from lamindb_setup.core._settings import settings as setup_settings
 from lamindb_setup.core._settings_instance import sanitize_git_repo_url
 
 from .subsettings._creation_settings import CreationSettings, creation_settings
-from .subsettings._transform_settings import TransformSettings, transform_settings
 
 if TYPE_CHECKING:
     from collections.abc import Mapping
@@ -80,16 +79,6 @@ class Settings:
             storage_settings = ln_setup.core.StorageSettings(root=self._using_storage)
         return storage_settings
 
-    @property
-    def transform(self) -> TransformSettings:
-        """Transform settings.
-
-        Is deprecated since version 0.76.1.
-        """
-        # enable warning soon
-        # logger.warning("Transform settings are deprecated, please instead set `ln.context.uid`")
-        return transform_settings
-
     @property
     def sync_git_repo(self) -> str | None:
         """Sync transforms with scripts in git repository.

lamindb/core/_sync_git.py

@@ -53,22 +53,69 @@ def check_local_git_repo() -> bool:
 
 
 def get_git_commit_hash(blob_hash: str, repo_dir: Path | None = None) -> str | None:
-    command = ["git", "log", f"--find-object={blob_hash}", "--pretty=format:%H"]
+    # Fetch all remote branches so that we can also search them
+    fetch_command = ["git", "fetch", "origin", "+refs/heads/*:refs/remotes/origin/*"]
+    subprocess.run(fetch_command, cwd=repo_dir, check=True)
+
+    # Find the commit containing the blob hash in all branches
+    command = [
+        "git",
+        "log",
+        "--all",
+        f"--find-object={blob_hash}",
+        "--pretty=format:%H",
+    ]
     result = subprocess.run(
         command,
         capture_output=True,
         cwd=repo_dir,
     )
-    # we just care to find one commit
-    # hence, we split by new line ("\n") and use the first one
+    # We just care to find one commit
+    # Hence, we split by new line ("\n") and use the first one
     commit_hash = result.stdout.decode().split("\n")[0]
-    if commit_hash == "" or result.returncode == 1:
+
+    if not commit_hash or result.returncode == 1:
         return None
-    else:
-        assert (  # noqa: S101
-            len(commit_hash) == 40
-        ), f"commit hash |{commit_hash}| is not 40 characters long"
-        return commit_hash
+
+    default_branch = (
+        subprocess.run(
+            ["git", "rev-parse", "--abbrev-ref", "origin/HEAD"],
+            capture_output=True,
+            cwd=repo_dir,
+            text=True,
+        )
+        .stdout.strip()
+        .split("/")[-1]
+    )
+
+    # Find all branches containing the commit
+    commit_containing_branches = subprocess.run(
+        ["git", "branch", "--all", "--contains", commit_hash],
+        capture_output=True,
+        cwd=repo_dir,
+        text=True,
+    ).stdout.split("\n")
+
+    # Clean up branch names and filter out the default branch
+    commit_containing_branches = [
+        branch.strip().replace("remotes/", "")
+        for branch in commit_containing_branches
+        if branch.strip()
+    ]
+    non_default_branches = [
+        branch for branch in commit_containing_branches if default_branch not in branch
+    ]
+
+    if non_default_branches:
+        logger.warning(
+            f"code blob hash {blob_hash} was found in non-default branch(es): {', '.join(non_default_branches)}"
+        )
+
+    assert (  # noqa: S101
+        len(commit_hash) == 40
+    ), f"commit hash |{commit_hash}| is not 40 characters long"
+
+    return commit_hash
 
 
 def get_filepath_within_git_repo(

lamindb/core/_track_environment.py

@@ -7,7 +7,7 @@ import lamindb_setup as ln_setup
 from lamin_utils import logger
 
 if TYPE_CHECKING:
-    from lnschema_core.models import Run
+    from lamindb.models import Run
 
 
 def track_environment(run: Run) -> None:

lamindb/core/datasets/_core.py

@@ -5,11 +5,10 @@ from typing import TYPE_CHECKING
 from urllib.request import urlretrieve
 
 import anndata as ad
-import numpy as np
 import pandas as pd
-from lnschema_core import ids
 from upath import UPath
 
+from lamindb.base.ids import base62
 from lamindb.core._settings import settings
 
 if TYPE_CHECKING:
@@ -146,7 +145,7 @@ def dir_iris_images() -> UPath:  # pragma: no cover
     This is why on the UI, the artifact shows up as output of the downstream
     demo notebook rather than the upstream curation notebook. The lineage
     information should still be captured by
-    https://github.com/laminlabs/lnschema-core/blob/a90437e91dfbd6b9002f18c3e978bd0f9c9a632d/lnschema_core/models.py#L2050-L2052
+    https://github.com/laminlabs/lnschema-core/blob/a90437e91dfbd6b9002f18c3e978bd0f9c9a632d/lamindb/models.py#L2050-L2052
     but we don't use this in the UI yet.
     """
     return UPath("s3://lamindata/iris_studies")
@@ -481,11 +480,11 @@ def dir_scrnaseq_cellranger(
         fastqdir.mkdir(parents=True, exist_ok=True)
         fastqfile1 = fastqdir / f"{sample_name}_R1_001.fastq.gz"
         with open(fastqfile1, "w") as f:
-            f.write(f"{ids.base62(n_char=6)}")
+            f.write(f"{base62(n_char=6)}")
         fastqfile2 = fastqdir / f"{sample_name}_R2_001.fastq.gz"
         fastqfile2.touch(exist_ok=True)
         with open(fastqfile2, "w") as f:
-            f.write(f"{ids.base62(n_char=6)}")
+            f.write(f"{base62(n_char=6)}")
 
     sampledir = basedir / f"{sample_name}"
     for folder in ["raw_feature_bc_matrix", "filtered_feature_bc_matrix", "analysis"]:
@@ -511,7 +510,7 @@ def dir_scrnaseq_cellranger(
     ]:
         file = sampledir / filename
         with open(file, "w") as f:
-            f.write(f"{ids.base62(n_char=6)}")
+            f.write(f"{base62(n_char=6)}")
 
     return sampledir
 

lamindb/core/exceptions.py

@@ -7,7 +7,6 @@
    DoesNotExist
    ValidationError
    NotebookNotSaved
-   NoTitleError
    MissingContextUID
    UpdateContext
    IntegrityError
@@ -79,12 +78,6 @@ class IntegrityError(Exception):
     pass
 
 
-class NoTitleError(SystemExit):
-    """Notebook has no title."""
-
-    pass
-
-
 class MissingContextUID(SystemExit):
     """User didn't define transform settings."""
 

lamindb/core/fields.py

@@ -9,4 +9,4 @@ The field accessor of a :class:`~lamindb.core.Record`:
 
 """
 
-from lnschema_core.types import FieldAttr
+from lamindb.base.types import FieldAttr  # noqa: F401

lamindb/core/loaders.py

@@ -33,7 +33,6 @@ from lamindb_setup.core.upath import (
 from ._settings import settings
 
 if TYPE_CHECKING:
-    import mudata as md
     from lamindb_setup.core.types import UPathStr
 
 try:

lamindb/core/{schema.py → relations.py}

@@ -7,47 +7,50 @@ from lamindb_setup._connect_instance import (
     load_instance_settings,
 )
 from lamindb_setup.core._settings_store import instance_settings_file
-from lnschema_core.models import Feature, FeatureSet, LinkORM, Record
 
+from lamindb.models import LinkORM, Record, Schema
 
-def get_schemas_modules(instance: str | None) -> set[str]:
+
+def get_schema_modules(instance: str | None) -> set[str]:
     if instance is None or instance == "default":
-        schema_modules = set(ln_setup.settings.instance.schema)
+        schema_modules = set(ln_setup.settings.instance.modules)
         schema_modules.add("core")
         return schema_modules
     owner, name = get_owner_name_from_identifier(instance)
     settings_file = instance_settings_file(name, owner)
     if settings_file.exists():
-        schema = set(load_instance_settings(settings_file).schema)
+        modules = set(load_instance_settings(settings_file).modules)
     else:
         cache_filepath = (
             ln_setup.settings.cache_dir / f"instance--{owner}--{name}--uid.txt"
         )
         if cache_filepath.exists():
-            schema = set(cache_filepath.read_text().split("\n")[1].split(","))
+            modules = set(cache_filepath.read_text().split("\n")[1].split(","))
         else:
             raise ValueError(f"Instance {instance} not found")
-    shared_schema_modules = set(ln_setup.settings.instance.schema).intersection(schema)
+    shared_schema_modules = set(ln_setup.settings.instance.modules).intersection(
+        modules
+    )
     shared_schema_modules.add("core")
     return shared_schema_modules
 
 
-def dict_schema_name_to_model_name(
+def dict_module_name_to_model_name(
     registry: type[Record], instance: str | None = None
 ) -> dict[str, Record]:
-    schema_modules = get_schemas_modules(instance)
+    schema_modules = get_schema_modules(instance)
     d: dict = {
-        i.related_model.__get_name_with_schema__(): i.related_model
+        i.related_model.__get_name_with_module__(): i.related_model
         for i in registry._meta.related_objects
         if i.related_name is not None
-        and i.related_model.__get_schema_name__() in schema_modules
+        and i.related_model.__get_module_name__() in schema_modules
     }
     d.update(
         {
-            i.related_model.__get_name_with_schema__(): i.related_model
+            i.related_model.__get_name_with_module__(): i.related_model
             for i in registry._meta.many_to_many
             if i.name is not None
-            and i.related_model.__get_schema_name__() in schema_modules
+            and i.related_model.__get_module_name__() in schema_modules
         }
     )
     return d
@@ -59,11 +62,11 @@ def dict_related_model_to_related_name(
     def include(model: Record):
         return not links != issubclass(model, LinkORM)
 
-    schema_modules = get_schemas_modules(instance)
+    schema_modules = get_schema_modules(instance)
 
     related_objects = registry._meta.related_objects + registry._meta.many_to_many
     d: dict = {
-        record.related_model.__get_name_with_schema__(): (
+        record.related_model.__get_name_with_module__(): (
             record.related_name
             if not isinstance(record, ManyToManyField)
             else record.name
@@ -72,7 +75,7 @@ def dict_related_model_to_related_name(
         if (
             record.name is not None
             and include(record.related_model)
-            and record.related_model.__get_schema_name__() in schema_modules
+            and record.related_model.__get_module_name__() in schema_modules
         )
     }
     return d
@@ -81,15 +84,15 @@ def dict_related_model_to_related_name(
 def get_related_name(features_type: type[Record]) -> str:
     candidates = [
         field.related_name
-        for field in FeatureSet._meta.related_objects
+        for field in Schema._meta.related_objects
         if field.related_model == features_type
     ]
     if not candidates:
         raise ValueError(
             f"Can't create feature sets from {features_type.__name__} because it's not"
-            " related to it!\nYou need to create a link model between FeatureSet and"
-            " your Record in your custom schema.\nTo do so, add a"
-            " line:\nfeature_sets = models.ManyToMany(FeatureSet,"
+            " related to it!\nYou need to create a link model between Schema and"
+            " your Record in your custom module.\nTo do so, add a"
+            " line:\n_schemas_m2m = models.ManyToMany(Schema,"
             " related_name='mythings')\n"
         )
     return candidates[0]

lamindb/core/storage/_anndata_accessor.py

@@ -17,12 +17,11 @@ from anndata._io.specs.registry import get_spec, read_elem, read_elem_partial
 from anndata.compat import _read_attr
 from fsspec.implementations.local import LocalFileSystem
 from lamin_utils import logger
-from lamindb_setup.core.upath import UPath, create_mapper, infer_filesystem
+from lamindb_setup.core.upath import create_mapper, infer_filesystem
 from packaging import version
 
 if TYPE_CHECKING:
     from collections.abc import Mapping
-    from pathlib import Path
 
     from fsspec.core import OpenFile
     from lamindb_setup.core.types import UPathStr

lamindb/core/storage/_backed_access.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Callable
 
 from anndata._io.specs.registry import get_spec
-from lnschema_core import Artifact
+
+from lamindb.models import Artifact
 
 from ._anndata_accessor import AnnDataAccessor, StorageType, registry
 from ._pyarrow_dataset import _is_pyarrow_dataset, _open_pyarrow_dataset

lamindb/core/storage/_tiledbsoma.py

@@ -2,12 +2,16 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Literal
 
+import pandas as pd
+import pyarrow as pa
 from anndata import AnnData, read_h5ad
 from lamin_utils import logger
 from lamindb_setup import settings as setup_settings
 from lamindb_setup.core._settings_storage import get_storage_region
 from lamindb_setup.core.upath import LocalPathClasses, create_path
-from lnschema_core import Artifact, Run
+from packaging import version
+
+from lamindb.models import Artifact, Run
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
@@ -138,9 +142,17 @@ def save_tiledbsoma_experiment(
     storepath = storepath.as_posix()
 
     add_run_uid = True
+    run_uid_dtype = "category"
     if appending:
         with soma.Experiment.open(storepath, mode="r", context=ctx) as store:
-            add_run_uid = "lamin_run_uid" in store["obs"].schema.names
+            obs_schema = store["obs"].schema
+            add_run_uid = "lamin_run_uid" in obs_schema.names
+            # this is needed to enable backwards compatibility with tiledbsoma stores
+            # created before PR 2300
+            if add_run_uid:
+                column_type = obs_schema.types[obs_schema.names.index("lamin_run_uid")]
+                if not isinstance(column_type, pa.DictionaryType):
+                    run_uid_dtype = None
 
     if add_run_uid and run is None:
         raise ValueError("Pass `run`")
@@ -148,17 +160,16 @@ def save_tiledbsoma_experiment(
     adata_objects = []
     for adata in adatas:
         if isinstance(adata, AnnData):
-            if add_run_uid:
-                if adata.is_view:
-                    raise ValueError(
-                        "Can not write an `AnnData` view, please do `adata.copy()` before passing."
-                    )
-                else:
-                    adata.obs["lamin_run_uid"] = run.uid
+            if add_run_uid and adata.is_view:
+                raise ValueError(
+                    "Can not write an `AnnData` view, please do `adata.copy()` before passing."
+                )
         else:
             adata = _load_h5ad_zarr(create_path(adata))
-            if add_run_uid:
-                adata.obs["lamin_run_uid"] = run.uid
+        if add_run_uid:
+            adata.obs["lamin_run_uid"] = pd.Series(
+                run.uid, index=adata.obs.index, dtype=run_uid_dtype
+            )
         adata_objects.append(adata)
 
     registration_mapping = kwargs.get("registration_mapping", None)
@@ -173,14 +184,28 @@ def save_tiledbsoma_experiment(
             context=ctx,
         )
 
+    resize_experiment = False
     if registration_mapping is not None:
-        n_observations = len(registration_mapping.obs_axis.data)
+        if version.parse(soma.__version__) < version.parse("1.15.0rc4"):
+            n_observations = len(registration_mapping.obs_axis.data)
+        else:
+            n_observations = registration_mapping.get_obs_shape()
+            resize_experiment = True
     else:  # happens only if not appending and only one adata passed
         assert len(adata_objects) == 1  # noqa: S101
         n_observations = adata_objects[0].n_obs
 
     logger.important(f"Writing the tiledbsoma store to {storepath}")
     for adata_obj in adata_objects:
+        if resize_experiment and soma.Experiment.exists(storepath, context=ctx):
+            # can only happen if registration_mapping is not None
+            soma_io.resize_experiment(
+                storepath,
+                nobs=n_observations,
+                nvars=registration_mapping.get_var_shapes(),
+                context=ctx,
+            )
+            resize_experiment = False
         soma_io.from_anndata(
             storepath,
             adata_obj,
@@ -201,6 +226,6 @@ def save_tiledbsoma_experiment(
         _is_internal_call=True,
     )
     artifact.n_observations = n_observations
-    artifact._accessor = "tiledbsoma"
+    artifact.otype = "tiledbsoma"
 
     return artifact.save()

lamindb/core/storage/objects.py

@@ -12,7 +12,7 @@ if TYPE_CHECKING:
 
 def _mudata_is_installed():
     try:
-        import mudata
+        import mudata  # noqa: F401c
     except ImportError:
         return False
     return True

lamindb/core/storage/paths.py

@@ -3,20 +3,16 @@ from __future__ import annotations
 import shutil
 from typing import TYPE_CHECKING
 
-import anndata as ad
 import fsspec
-import pandas as pd
 from lamin_utils import logger
 from lamindb_setup.core import StorageSettings
 from lamindb_setup.core.upath import (
     LocalPathClasses,
     UPath,
-    create_path,
-    infer_filesystem,
 )
-from lnschema_core.models import Artifact, Storage
 
 from lamindb.core._settings import settings
+from lamindb.models import Artifact, Storage
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -30,7 +26,7 @@ AUTO_KEY_PREFIX = ".lamindb/"
 # add type annotations back asap when re-organizing the module
 def auto_storage_key_from_artifact(artifact: Artifact):
     if artifact.key is None or artifact._key_is_virtual:
-        is_dir = artifact.n_objects is not None
+        is_dir = artifact.n_files is not None
         return auto_storage_key_from_artifact_uid(artifact.uid, artifact.suffix, is_dir)
     else:
         return artifact.key
@@ -52,10 +48,19 @@ def check_path_is_child_of_root(path: UPathStr, root: UPathStr) -> bool:
     # and for fsspec.utils.get_protocol
     path_str = str(path)
     root_str = str(root)
+    root_protocol = fsspec.utils.get_protocol(root_str)
     # check that the protocols are the same first
-    if fsspec.utils.get_protocol(path_str) != fsspec.utils.get_protocol(root_str):
+    if fsspec.utils.get_protocol(path_str) != root_protocol:
         return False
-    return UPath(root_str).resolve() in UPath(path_str).resolve().parents
+    if root_protocol in {"http", "https"}:
+        # in this case it is a base url, not a file
+        # so formally does not exist
+        resolve_kwargs = {"follow_redirects": False}
+    else:
+        resolve_kwargs = {}
+    return (
+        UPath(root_str).resolve(**resolve_kwargs) in UPath(path_str).resolve().parents
+    )
 
 
 # returns filepath and root of the storage

lamindb/core/subsettings/__init__.py

@@ -3,10 +3,8 @@
 .. autosummary::
    :toctree: .
 
-   TransformSettings
    CreationSettings
 
 """
 
 from ._creation_settings import CreationSettings
-from ._transform_settings import TransformSettings