lamindb 0.63.5__py3-none-any.whl → 0.64.1__py3-none-any.whl

lamindb/_save.py

@@ -11,12 +11,12 @@ from django.db import transaction
 from django.utils.functional import partition
 from lamin_utils import logger
 from lamindb_setup.dev.upath import print_hook
-from lnschema_core.models import File, Registry
+from lnschema_core.models import Artifact, Registry
 
-from lamindb.dev.storage import store_object
 from lamindb.dev.storage.file import (
-    auto_storage_key_from_file,
+    auto_storage_key_from_artifact,
     delete_storage_using_key,
+    store_artifact,
 )
 
 try:
@@ -73,20 +73,20 @@ def save(
 
     # previously, this was all set based,
     # but models without primary keys aren't hashable
-    # we distinguish between files and non-files
-    # for files, we want to bulk-upload
+    # we distinguish between artifacts and non-artifacts
+    # for artifacts, we want to bulk-upload
     # rather than upload one-by-one
-    non_files, files = partition(lambda r: isinstance(r, File), records)
-    if non_files:
+    non_artifacts, artifacts = partition(lambda r: isinstance(r, Artifact), records)
+    if non_artifacts:
         # first save all records that do not yet have a primary key without
         # recursing parents
-        _, non_files_without_pk = partition(lambda r: r.pk is None, non_files)
-        bulk_create(non_files_without_pk, ignore_conflicts=ignore_conflicts)
-        non_files_with_parents = [
-            r for r in non_files_without_pk if hasattr(r, "_parents")
+        _, non_artifacts_without_pk = partition(lambda r: r.pk is None, non_artifacts)
+        bulk_create(non_artifacts_without_pk, ignore_conflicts=ignore_conflicts)
+        non_artifacts_with_parents = [
+            r for r in non_artifacts_without_pk if hasattr(r, "_parents")
         ]
 
-        if len(non_files_with_parents) > 0 and kwargs.get("parents") is not False:
+        if len(non_artifacts_with_parents) > 0 and kwargs.get("parents") is not False:
             # this can only happen within lnschema_bionty right now!!
             # we might extend to core lamindb later
             import lnschema_bionty as lb
@@ -105,14 +105,14 @@ def save(
                         "you can switch this off via: lb.settings.auto_save_parents ="
                         " False"
                     )
-                for record in non_files_with_parents:
+                for record in non_artifacts_with_parents:
                     record._save_ontology_parents(mute=True)
 
-    if files:
+    if artifacts:
         with transaction.atomic():
-            for record in files:
+            for record in artifacts:
                 record._save_skip_storage()
-        store_files(files)
+        store_artifacts(artifacts)
 
     # this function returns None as potentially 10k records might be saved
     # refreshing all of them from the DB would mean a severe performance penalty
@@ -128,27 +128,27 @@ def bulk_create(records: Iterable[Registry], ignore_conflicts: Optional[bool] =
         orm.objects.bulk_create(records, ignore_conflicts=ignore_conflicts)
 
 
-# This is also used within File.save()
-def check_and_attempt_upload(file: File) -> Optional[Exception]:
-    # if File object is either newly instantiated or replace() was called on
+# This is also used within Artifact.save()
+def check_and_attempt_upload(artifact: Artifact) -> Optional[Exception]:
+    # if Artifact object is either newly instantiated or replace() was called on
     # a local env it will have a _local_filepath and needs to be uploaded
-    if hasattr(file, "_local_filepath"):
+    if hasattr(artifact, "_local_filepath"):
         try:
-            upload_data_object(file)
+            upload_artifact(artifact)
         except Exception as exception:
-            logger.warning(f"could not upload file: {file}")
+            logger.warning(f"could not upload artifact: {artifact}")
             return exception
         # copies (if on-disk) or moves the temporary file (if in-memory) to the cache
-        copy_or_move_to_cache(file)
+        copy_or_move_to_cache(artifact)
         # after successful upload, we should remove the attribute so that another call
         # call to save won't upload again, the user should call replace() then
-        del file._local_filepath
+        del artifact._local_filepath
     # returning None means proceed (either success or no action needed)
     return None
 
 
-def copy_or_move_to_cache(file: File):
-    local_path = file._local_filepath
+def copy_or_move_to_cache(artifact: Artifact):
+    local_path = artifact._local_filepath
 
     # in-memory zarr or on-disk zarr
     if local_path is None or not local_path.is_file():
@@ -164,7 +164,7 @@ def copy_or_move_to_cache(file: File):
         return None
 
     # maybe create something like storage.key_to_local(key) later to simplfy
-    storage_key = auto_storage_key_from_file(file)
+    storage_key = auto_storage_key_from_artifact(artifact)
     storage_path = lamindb_setup.settings.storage.key_to_filepath(storage_key)
     cache_path = lamindb_setup.settings.storage.cloud_to_local_no_update(storage_path)
     cache_path.parent.mkdir(parents=True, exist_ok=True)
@@ -178,58 +178,58 @@ def copy_or_move_to_cache(file: File):
     os.utime(cache_path, times=(mts, mts))
 
 
-# This is also used within File.save()
-def check_and_attempt_clearing(file: File) -> Optional[Exception]:
+# This is also used within Artifact.save()
+def check_and_attempt_clearing(artifact: Artifact) -> Optional[Exception]:
     # this is a clean-up operation after replace() was called
     # this will only evaluate to True if replace() was called
-    if hasattr(file, "_clear_storagekey"):
+    if hasattr(artifact, "_clear_storagekey"):
         try:
-            if file._clear_storagekey is not None:
-                delete_storage_using_key(file, file._clear_storagekey)
+            if artifact._clear_storagekey is not None:
+                delete_storage_using_key(artifact, artifact._clear_storagekey)
                 logger.success(
-                    f"deleted stale object at storage key {file._clear_storagekey}"
+                    f"deleted stale object at storage key {artifact._clear_storagekey}"
                 )
-                file._clear_storagekey = None
+                artifact._clear_storagekey = None
         except Exception as exception:
             return exception
     # returning None means proceed (either success or no action needed)
     return None
 
 
-def store_files(files: Iterable[File]) -> None:
-    """Upload files in a list of database-committed files to storage.
+def store_artifacts(artifacts: Iterable[Artifact]) -> None:
+    """Upload artifacts in a list of database-committed artifacts to storage.
 
-    If any upload fails, subsequent files are cleaned up from the DB.
+    If any upload fails, subsequent artifacts are cleaned up from the DB.
     """
     exception: Optional[Exception] = None
     # because uploads might fail, we need to maintain a new list
     # of the succeeded uploads
-    stored_files = []
+    stored_artifacts = []
 
-    # upload new local files
-    for file in files:
-        exception = check_and_attempt_upload(file)
+    # upload new local artifacts
+    for artifact in artifacts:
+        exception = check_and_attempt_upload(artifact)
         if exception is not None:
             break
-        stored_files += [file]
-        exception = check_and_attempt_clearing(file)
+        stored_artifacts += [artifact]
+        exception = check_and_attempt_clearing(artifact)
         if exception is not None:
-            logger.warning(f"clean up of {file._clear_storagekey} failed")
+            logger.warning(f"clean up of {artifact._clear_storagekey} failed")
             break
 
     if exception is not None:
-        # clean up metadata for files not uploaded to storage
+        # clean up metadata for artifacts not uploaded to storage
         with transaction.atomic():
-            for file in files:
-                if file not in stored_files:
-                    file._delete_skip_storage()
-        error_message = prepare_error_message(files, stored_files, exception)
+            for artifact in artifacts:
+                if artifact not in stored_artifacts:
+                    artifact._delete_skip_storage()
+        error_message = prepare_error_message(artifacts, stored_artifacts, exception)
         raise RuntimeError(error_message)
     return None
 
 
-def prepare_error_message(records, stored_files, exception) -> str:
-    if len(records) == 1 or len(stored_files) == 0:
+def prepare_error_message(records, stored_artifacts, exception) -> str:
+    if len(records) == 1 or len(stored_artifacts) == 0:
         error_message = (
             "No entries were uploaded or committed"
             " to the database. See error message:\n\n"
@@ -239,7 +239,7 @@ def prepare_error_message(records, stored_files, exception) -> str:
             "The following entries have been"
             " successfully uploaded and committed to the database:\n"
         )
-        for record in stored_files:
+        for record in stored_artifacts:
             error_message += (
                 f"- {', '.join(record.__repr__().split(', ')[:3]) + ', ...)'}\n"
             )
@@ -248,24 +248,24 @@ def prepare_error_message(records, stored_files, exception) -> str:
     return error_message
 
 
-def upload_data_object(file) -> None:
+def upload_artifact(artifact) -> None:
     """Store and add file and its linked entries."""
     # do NOT hand-craft the storage key!
-    file_storage_key = auto_storage_key_from_file(file)
+    artifact_storage_key = auto_storage_key_from_artifact(artifact)
     storage_path = lamindb_setup.settings.instance.storage.key_to_filepath(
-        file_storage_key
+        artifact_storage_key
     )
-    msg = f"storing file '{file.uid}' at '{storage_path}'"
+    msg = f"storing artifact '{artifact.uid}' at '{storage_path}'"
     if (
-        file.suffix in {".zarr", ".zrad"}
-        and hasattr(file, "_memory_rep")
-        and file._memory_rep is not None
+        artifact.suffix in {".zarr", ".zrad"}
+        and hasattr(artifact, "_memory_rep")
+        and artifact._memory_rep is not None
     ):
         logger.save(msg)
         print_progress = partial(
-            print_hook, filepath=file_storage_key, action="uploading"
+            print_hook, filepath=artifact_storage_key, action="uploading"
         )
-        write_adata_zarr(file._memory_rep, storage_path, callback=print_progress)
-    elif hasattr(file, "_to_store") and file._to_store:
+        write_adata_zarr(artifact._memory_rep, storage_path, callback=print_progress)
+    elif hasattr(artifact, "_to_store") and artifact._to_store:
         logger.save(msg)
-        store_object(file._local_filepath, file_storage_key)
+        store_artifact(artifact._local_filepath, artifact_storage_key)

lamindb/dev/_data.py

@@ -4,11 +4,11 @@ from typing import Any, Dict, Iterable, List, Optional, Union
 from lamin_utils import colors, logger
 from lamindb_setup.dev._docs import doc_args
 from lnschema_core.models import (
+    Artifact,
     Data,
     Dataset,
     Feature,
     FeatureSet,
-    File,
     Registry,
     Run,
     ULabel,
@@ -54,7 +54,7 @@ def add_transform_to_kwargs(kwargs: Dict[str, Any], run: Run):
         kwargs["transform"] = run.transform
 
 
-def save_feature_sets(self: Union[File, Dataset]) -> None:
+def save_feature_sets(self: Union[Artifact, Dataset]) -> None:
     if hasattr(self, "_feature_sets"):
         saved_feature_sets = {}
         for key, feature_set in self._feature_sets.items():
@@ -72,7 +72,7 @@ def save_feature_sets(self: Union[File, Dataset]) -> None:
             )
 
 
-def save_feature_set_links(self: Union[File, Dataset]) -> None:
+def save_feature_set_links(self: Union[Artifact, Dataset]) -> None:
     from lamindb._save import bulk_create
 
     Data = self.__class__
@@ -116,7 +116,7 @@ def describe(self: Data):
         "initial_version": "🔖",
         "file": "📄",
     }
-    if len(foreign_key_fields) > 0:  # always True for File and Dataset
+    if len(foreign_key_fields) > 0:  # always True for Artifact and Dataset
         record_msg = f"{colors.green(model_name)}{__repr__(self, include_foreign_keys=False).lstrip(model_name)}"  # noqa
         msg += f"{record_msg}\n\n"
 
@@ -229,7 +229,7 @@ def add_labels(
                 " feature=ln.Feature(name='my_feature'))"
             )
         if feature.registries is not None:
-            orm_dict = dict_schema_name_to_model_name(File)
+            orm_dict = dict_schema_name_to_model_name(Artifact)
             for reg in feature.registries.split("|"):
                 orm = orm_dict.get(reg)
                 records_validated += orm.from_values(records, field=field)
@@ -398,14 +398,15 @@ def _track_run_input(
         if run is None:
             raise ValueError(
                 "No run context set. Call ln.track() or link input to a"
-                " run object via `run.input_files.add(file)`"
+                " run object via `run.input_artifacts.add(artifact)`"
             )
         # avoid adding the same run twice
         run.save()
-        if data_class_name == "file":
-            LinkORM = run.input_files.through
+        if data_class_name == "artifact":
+            LinkORM = run.input_artifacts.through
             links = [
-                LinkORM(run_id=run.id, file_id=data_id) for data_id in input_data_ids
+                LinkORM(run_id=run.id, artifact_id=data_id)
+                for data_id in input_data_ids
             ]
         else:
             LinkORM = run.input_datasets.through

lamindb/dev/_feature_manager.py

@@ -2,7 +2,7 @@ from typing import Dict, Union
 
 import numpy as np
 from lamin_utils import colors
-from lnschema_core.models import Data, Dataset, Feature, File
+from lnschema_core.models import Artifact, Data, Dataset, Feature
 
 from .._feature_set import FeatureSet
 from .._query_set import QuerySet
@@ -15,15 +15,15 @@ from .._registry import (
 from .._save import save
 
 
-def get_host_id_field(host: Union[File, Dataset]) -> str:
-    if isinstance(host, File):
-        host_id_field = "file_id"
+def get_host_id_field(host: Union[Artifact, Dataset]) -> str:
+    if isinstance(host, Artifact):
+        host_id_field = "artifact_id"
     else:
         host_id_field = "dataset_id"
     return host_id_field
 
 
-def get_accessor_by_orm(host: Union[File, Dataset]) -> Dict:
+def get_accessor_by_orm(host: Union[Artifact, Dataset]) -> Dict:
     dictionary = {
         field.related_model.__get_name_with_schema__(): field.name
         for field in host._meta.related_objects
@@ -56,7 +56,7 @@ def get_feature_set_by_slot(host) -> Dict:
 
 
 def get_label_links(
-    host: Union[File, Dataset], registry: str, feature: Feature
+    host: Union[Artifact, Dataset], registry: str, feature: Feature
 ) -> QuerySet:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id, "feature_id": feature.id}
@@ -68,7 +68,7 @@ def get_label_links(
     return link_records
 
 
-def get_feature_set_links(host: Union[File, Dataset]) -> QuerySet:
+def get_feature_set_links(host: Union[Artifact, Dataset]) -> QuerySet:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id}
     feature_set_links = host.feature_sets.through.objects.filter(**kwargs)
@@ -124,7 +124,7 @@ class FeatureManager:
     See :class:`~lamindb.dev.Data` for more information.
     """
 
-    def __init__(self, host: Union[File, Dataset]):
+    def __init__(self, host: Union[Artifact, Dataset]):
         self._host = host
         self._feature_set_by_slot = get_feature_set_by_slot(host)
         self._accessor_by_orm = get_accessor_by_orm(host)
@@ -160,7 +160,7 @@ class FeatureManager:
         """
         if self._host._state.adding:
             raise ValueError(
-                "Please save the file or dataset before adding a feature set!"
+                "Please save the artifact or dataset before adding a feature set!"
             )
         host_db = self._host._state.db
         feature_set.save(using=host_db)
@@ -180,7 +180,7 @@ class FeatureManager:
             self._feature_set_by_slot[slot] = feature_set
 
     def _add_from(self, data: Data):
-        """Transfer features from a file or dataset."""
+        """Transfer features from a artifact or dataset."""
         for slot, feature_set in data.features._feature_set_by_slot.items():
             members = feature_set.members
             registry = members[0].__class__

lamindb/dev/_label_manager.py

@@ -2,7 +2,7 @@ from typing import Dict, List, Optional, Union
 
 import numpy as np
 from lamin_utils import colors, logger
-from lnschema_core.models import Data, Dataset, Feature, File, Registry
+from lnschema_core.models import Artifact, Data, Dataset, Feature, Registry
 
 from .._feature_set import dict_related_model_to_related_name
 from .._from_values import _print_values
@@ -103,7 +103,7 @@ class LabelManager:
     See :class:`~lamindb.dev.Data` for more information.
     """
 
-    def __init__(self, host: Union[File, Dataset]):
+    def __init__(self, host: Union[Artifact, Dataset]):
         self._host = host
 
     def __repr__(self) -> str:
@@ -150,9 +150,9 @@ class LabelManager:
         """Transfer labels from a file or dataset.
 
         Examples:
-            >>> file1 = ln.File(pd.DataFrame(index=[0, 1]))
+            >>> file1 = ln.Artifact(pd.DataFrame(index=[0, 1]))
             >>> file1.save()
-            >>> file2 = ln.File(pd.DataFrame(index=[2, 3]))
+            >>> file2 = ln.Artifact(pd.DataFrame(index=[2, 3]))
             >>> file2.save()
             >>> ulabels = ln.ULabel.from_values(["Label1", "Label2"], field="name")
             >>> ln.save(ulabels)

lamindb/dev/_run_context.py

@@ -555,7 +555,7 @@ class run_context:
         else:
             # check whether there was an update
             if (
-                transform.source_file_id is not None
+                transform.source_code_id is not None
                 or transform.latest_report_id is not None
             ):
                 if os.getenv("LAMIN_TESTING") is None:
@@ -572,7 +572,7 @@ class run_context:
                 else:
                     logger.warning(
                         "not tracking this transform, either increase version or delete"
-                        " the saved transform.source_file and transform.latest_report"
+                        " the saved transform.source_code and transform.latest_report"
                     )
                     return False
             if transform.name != name or transform.short_name != short_name:

lamindb/dev/_settings.py

@@ -29,7 +29,7 @@ class Settings:
         self._verbosity_int: int = 1  # success-level logging
         logger.set_verbosity(self._verbosity_int)
 
-    upon_file_create_if_hash_exists: Literal[
+    upon_artifact_create_if_hash_exists: Literal[
         "warn_return_existing", "error", "warn_create_new"
     ] = "warn_return_existing"
     """Behavior if file hash exists (default `"warn_return_existing"`).
@@ -61,10 +61,11 @@ class Settings:
     """
     silence_file_run_transform_warning: bool = False
     """Silence warning about missing run & transform during file creation."""
-    file_use_virtual_keys: bool = True
-    """The `key` parameter in :class:`~lamindb.File` is treated as a virtual storage key.
+    artifact_use_virtual_keys: bool = True
+    """Treat `key` parameter in :class:`~lamindb.Artifact` as virtual.
 
-    If `True`, the `key` is **not** used to construct file paths.
+    If `True`, the `key` is **not** used to construct file paths, but file paths are
+    based on the `uid` of artifact.
     """
 
     @property

lamindb/dev/_view_tree.py

@@ -3,7 +3,7 @@ from collections import defaultdict
 from typing import Iterable
 
 from lamindb_setup import settings as setup_settings
-from lnschema_core.models import File, Storage
+from lnschema_core.models import Artifact, Storage
 
 
 def view_tree(
@@ -18,7 +18,7 @@ def view_tree(
         print("queryset")
         qs = cls
         storage_ids = qs.list("storage_id")
-    elif cls == File:
+    elif cls == Artifact:
         print("file")
         qs = cls.filter(storage_id=setup_settings.storage.id).all()
         storage_ids = Storage.filter().list("id")
@@ -30,9 +30,9 @@ def view_tree(
         storage_id: storages.get(id=storage_id).root for storage_id in storage_ids
     }
     keys = set()
-    for file in qs:
-        root = storage_roots.get(file.storage_id, "")
-        keys.add(f"{root}/{file.key}")
+    for artifact in qs:
+        root = storage_roots.get(artifact.storage_id, "")
+        keys.add(f"{root}/{artifact.key}")
 
     _view_tree(
         keys=keys,

lamindb/dev/datasets/_core.py

@@ -12,7 +12,7 @@ from .._settings import settings
 
 
 def file_fcs() -> Path:
-    """Example FCS file."""
+    """Example FCS artifact."""
     filepath, _ = urlretrieve(
         "https://lamindb-test.s3.amazonaws.com/example.fcs", "example.fcs"
     )
@@ -93,25 +93,25 @@ def file_tsv_rnaseq_nfcore_salmon_merged_gene_counts(
 
 
 def file_fastq(in_storage_root=False) -> Path:
-    """Mini mock fastq file."""
+    """Mini mock fastq artifact."""
     basedir = Path(".") if not in_storage_root else settings.storage
     filepath = basedir / "input.fastq.gz"
     with open(filepath, "w") as f:
-        f.write("Mock fastq file.")
+        f.write("Mock fastq artifact.")
     return filepath
 
 
 def file_bam(in_storage_root=False) -> Path:  # pragma: no cover
-    """Mini mock bam file."""
+    """Mini mock bam artifact."""
     basedir = Path(".") if not in_storage_root else settings.storage
     filepath = basedir / "output.bam"
     with open(filepath, "w") as f:
-        f.write("Mock bam file.")
+        f.write("Mock bam artifact.")
     return filepath
 
 
 def file_mini_csv(in_storage_root=False) -> Path:
-    """Mini csv file."""
+    """Mini csv artifact."""
     basedir = Path(".") if not in_storage_root else settings.storage
     filepath = basedir / "mini.csv"
     df = pd.DataFrame([1, 2, 3], columns=["test"])

lamindb/dev/hashing.py

@@ -10,7 +10,7 @@
 
 import base64
 import hashlib
-from typing import Set, Tuple
+from typing import List, Set, Tuple
 
 
 def to_b64_str(bstr: bytes):
@@ -29,6 +29,16 @@ def hash_set(s: Set[str]) -> str:
     return to_b64_str(hashlib.md5(bstr).digest())[:20]
 
 
+def hash_md5s_from_dir(etags: List[str]) -> Tuple[str, str]:
+    # need to sort below because we don't want the order of parsing the dir to
+    # affect the hash
+    digests = b"".join(
+        hashlib.md5(etag.encode("utf-8")).digest() for etag in sorted(etags)
+    )
+    digest = hashlib.md5(digests).digest()
+    return to_b64_str(digest)[:22], "md5-d"
+
+
 def hash_file(file_path, chunk_size=50 * 1024 * 1024) -> Tuple[str, str]:
     chunks = []
     with open(file_path, "rb") as fp:

lamindb/dev/storage/__init__.py

@@ -10,5 +10,5 @@ from lamindb_setup.dev.upath import LocalPathClasses, UPath, infer_filesystem
 
 from ._anndata_sizes import size_adata
 from ._backed_access import AnnDataAccessor, BackedAccessor
-from .file import delete_storage, load_to_memory, store_object
+from .file import delete_storage, load_to_memory
 from .object import infer_suffix, write_to_file

lamindb/dev/storage/_backed_access.py

@@ -19,10 +19,10 @@ from fsspec.core import OpenFile
 from fsspec.implementations.local import LocalFileSystem
 from lamin_utils import logger
 from lamindb_setup.dev.upath import UPath, infer_filesystem
-from lnschema_core import File
+from lnschema_core import Artifact
 from packaging import version
 
-from lamindb.dev.storage.file import filepath_from_file
+from lamindb.dev.storage.file import filepath_from_artifact
 
 anndata_version_parse = version.parse(anndata_version)
 
@@ -684,12 +684,12 @@ class BackedAccessor:
 
 
 def backed_access(
-    file_or_filepath: Union[File, Path]
+    artifact_or_filepath: Union[Artifact, Path]
 ) -> Union[AnnDataAccessor, BackedAccessor]:
-    if isinstance(file_or_filepath, File):
-        filepath = filepath_from_file(file_or_filepath)
+    if isinstance(artifact_or_filepath, Artifact):
+        filepath = filepath_from_artifact(artifact_or_filepath)
     else:
-        filepath = file_or_filepath
+        filepath = artifact_or_filepath
     name = filepath.name
 
     if filepath.suffix in (".h5", ".hdf5", ".h5ad"):