lamindb 1.0.5__py3-none-any.whl → 1.1.1__py3-none-any.whl

lamindb/_record.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import builtins
+import inspect
 import re
 from functools import reduce
 from pathlib import PurePosixPath
@@ -29,6 +30,7 @@ from django.db.models.lookups import (
     Regex,
     StartsWith,
 )
+from django.db.utils import IntegrityError
 from lamin_utils import colors, logger
 from lamin_utils._lookup import Lookup
 from lamindb_setup._connect_instance import (
@@ -41,7 +43,7 @@ from lamindb_setup.core._hub_core import connect_instance_hub
 from lamindb_setup.core._settings_store import instance_settings_file
 from lamindb_setup.core.upath import extract_suffix_from_path
 
-from lamindb.base.validation import FieldValidationError
+from lamindb.errors import FieldValidationError
 from lamindb.models import (
     Artifact,
     BasicRecord,
@@ -60,7 +62,7 @@ from lamindb.models import (
 
 from ._utils import attach_func_to_class_method
 from .core._settings import settings
-from .core.exceptions import (
+from .errors import (
     InvalidArgument,
     RecordNameChangeIntegrityError,
     ValidationError,
@@ -75,6 +77,30 @@ if TYPE_CHECKING:
 IPYTHON = getattr(builtins, "__IPYTHON__", False)
 
 
+def is_approx_pascal_case(s):
+    """Check if the last component of a dotted string is in PascalCase.
+
+    Args:
+        s (str): The string to check
+
+    Returns:
+        bool: True if the last component is in PascalCase
+
+    Raises:
+        ValueError: If the last component doesn't start with a capital letter
+    """
+    if "[" in s:  # this is because we allow types of form 'script[test_script.py]'
+        return True
+    last_component = s.split(".")[-1]
+
+    if not last_component[0].isupper():
+        raise ValueError(
+            f"'{last_component}' should start with a capital letter given you're defining a type"
+        )
+
+    return True
+
+
 def init_self_from_db(self: Record, existing_record: Record):
     new_args = [
         getattr(existing_record, field.attname) for field in self._meta.concrete_fields
@@ -86,7 +112,12 @@ def init_self_from_db(self: Record, existing_record: Record):
 
 def update_attributes(record: Record, attributes: dict[str, str]):
     for key, value in attributes.items():
-        if getattr(record, key) != value:
+        if (
+            getattr(record, key) != value
+            and value is not None
+            and key != "dtype"
+            and key != "_aux"
+        ):
             logger.warning(f"updated {key} from {getattr(record, key)} to {value}")
             setattr(record, key, value)
 
@@ -105,7 +136,7 @@ def validate_fields(record: Record, kwargs):
         k for k, v in kwargs.items() if v is None and k in required_fields
     ]
     if missing_fields:
-        raise TypeError(f"{missing_fields} are required.")
+        raise FieldValidationError(f"{missing_fields} are required.")
     # ensure the exact length of the internal uid for core entities
     if "uid" in kwargs and record.__class__ in {
         Artifact,
@@ -122,19 +153,36 @@ def validate_fields(record: Record, kwargs):
         ).max_length  # triggers FieldDoesNotExist
         if len(kwargs["uid"]) != uid_max_length:  # triggers KeyError
             raise ValidationError(
-                f'`uid` must be exactly {uid_max_length} characters long, got {len(kwargs["uid"])}.'
+                f"`uid` must be exactly {uid_max_length} characters long, got {len(kwargs['uid'])}."
+            )
+    # validate is_type
+    if "is_type" in kwargs and "name" in kwargs and kwargs["is_type"]:
+        if kwargs["name"].endswith("s"):
+            logger.warning(
+                f"name '{kwargs['name']}' for type ends with 's', in case you're naming with plural, consider the singular for a type name"
             )
+        is_approx_pascal_case(kwargs["name"])
     # validate literals
     validate_literal_fields(record, kwargs)
 
 
-def suggest_records_with_similar_names(record: Record, name_field: str, kwargs) -> bool:
+def suggest_records_with_similar_names(
+    record: Record, name_field: str, kwargs
+) -> Record | None:
     """Returns True if found exact match, otherwise False.
 
     Logs similar matches if found.
     """
     if kwargs.get(name_field) is None or not isinstance(kwargs.get(name_field), str):
-        return False
+        return None
+    # need to perform an additional request to find the exact match
+    # previously, this was inferred from the truncated/fuzzy search below
+    # but this isn't reliable: https://laminlabs.slack.com/archives/C04FPE8V01W/p1737812808563409
+    # the below needs to be .first() because there might be multiple records with the same
+    # name field in case the record is versioned (e.g. for Transform key)
+    exact_match = record.__class__.filter(**{name_field: kwargs[name_field]}).first()
+    if exact_match is not None:
+        return exact_match
     queryset = _search(
         record.__class__,
         kwargs[name_field],
@@ -143,10 +191,7 @@ def suggest_records_with_similar_names(record: Record, name_field: str, kwargs)
         limit=3,
     )
     if not queryset.exists():  # empty queryset
-        return False
-    for alternative_record in queryset:
-        if getattr(alternative_record, name_field) == kwargs[name_field]:
-            return True
+        return None
     s, it, nots = ("", "it", "s") if len(queryset) == 1 else ("s", "one of them", "")
     msg = f"record{s} with similar {name_field}{s} exist{nots}! did you mean to load {it}?"
     if IPYTHON:
@@ -157,11 +202,14 @@ def suggest_records_with_similar_names(record: Record, name_field: str, kwargs)
             display(queryset.df())
     else:
         logger.warning(f"{msg}\n{queryset}")
-    return False
+    return None
 
 
 def __init__(record: Record, *args, **kwargs):
-    if not args:
+    skip_validation = kwargs.pop("_skip_validation", False)
+    if not args and skip_validation:
+        super(BasicRecord, record).__init__(**kwargs)
+    elif not args and not skip_validation:
         validate_fields(record, kwargs)
 
         # do not search for names if an id is passed; this is important
@@ -170,15 +218,13 @@ def __init__(record: Record, *args, **kwargs):
         if "_has_consciously_provided_uid" in kwargs:
             has_consciously_provided_uid = kwargs.pop("_has_consciously_provided_uid")
         if (
-            isinstance(
-                record, (CanCurate, Collection, Transform)
-            )  # Collection is only temporary because it'll get a key field
+            isinstance(record, (CanCurate, Collection, Transform))
             and settings.creation.search_names
             and not has_consciously_provided_uid
         ):
             name_field = getattr(record, "_name_field", "name")
-            match = suggest_records_with_similar_names(record, name_field, kwargs)
-            if match:
+            exact_match = suggest_records_with_similar_names(record, name_field, kwargs)
+            if exact_match is not None:
                 if "version" in kwargs:
                     if kwargs["version"] is not None:
                         version_comment = " and version"
@@ -189,22 +235,25 @@ def __init__(record: Record, *args, **kwargs):
                             }
                         ).one_or_none()
                     else:
-                        # for a versioned record, an exact name match is not a
-                        # criterion for retrieving a record in case `version`
-                        # isn't passed - we'd always pull out many records with exactly the
-                        # same name
+                        # for a versioned record, an exact name match is not a criterion
+                        # for retrieving a record in case `version` isn't passed -
+                        # we'd always pull out many records with exactly the same name
                         existing_record = None
                 else:
                     version_comment = ""
-                    existing_record = record.__class__.filter(
-                        **{name_field: kwargs[name_field]}
-                    ).one_or_none()
+                    existing_record = exact_match
                 if existing_record is not None:
                     logger.important(
                         f"returning existing {record.__class__.__name__} record with same"
                         f" {name_field}{version_comment}: '{kwargs[name_field]}'"
                     )
+                    if isinstance(record, Schema):
+                        if existing_record.hash != kwargs["hash"]:
+                            raise ValueError(
+                                f"Schema name is already in use by schema with uid '{existing_record.uid}', please choose a different name."
+                            )
                     init_self_from_db(record, existing_record)
+                    update_attributes(record, kwargs)
                     return None
         super(BasicRecord, record).__init__(**kwargs)
         if isinstance(record, ValidateFields):
@@ -218,7 +267,9 @@ def __init__(record: Record, *args, **kwargs):
                 message = _format_django_validation_error(record, e)
                 raise FieldValidationError(message) from e
     elif len(args) != len(record._meta.concrete_fields):
-        raise ValueError("please provide keyword arguments, not plain arguments")
+        raise FieldValidationError(
+            f"Use keyword arguments instead of positional arguments, e.g.: {record.__class__.__name__}(name='...')."
+        )
     else:
         # object is loaded from DB (**kwargs could be omitted below, I believe)
         super(BasicRecord, record).__init__(*args, **kwargs)
@@ -257,6 +308,60 @@ def _format_django_validation_error(record: Record, e: DjangoValidationError):
         return message
 
 
+def _get_record_kwargs(record_class) -> list[tuple[str, str]]:
+    """Gets the parameters of a Record from the overloaded signature.
+
+    Example:
+        >>> get_record_params(bt.Organism)
+        >>> [('name', 'str'), ('taxon_id', 'str | None'), ('scientific_name', 'str | None')]
+    """
+    source = inspect.getsource(record_class)
+
+    # Find first overload that's not *db_args
+    pattern = r"@overload\s+def __init__\s*\(([\s\S]*?)\):\s*\.{3}"
+    overloads = re.finditer(pattern, source)
+
+    for overload in overloads:
+        params_block = overload.group(1)
+        # This is an additional safety measure if the overloaded signature that we're
+        # looking for is not at the top but a "db_args" constructor
+        if "*db_args" in params_block:
+            continue
+
+        params = []
+        for line in params_block.split("\n"):
+            line = line.strip()
+            if not line or "self" in line:
+                continue
+
+            # Extract name and type annotation
+            # The regex pattern finds parameter definitions like:
+            # Simple: name: str
+            # With default: age: int = 0
+            # With complex types: items: List[str] = []
+            param_pattern = (
+                r"(\w+)"  # Parameter name
+                r"\s*:\s*"  # Colon with optional whitespace
+                r"((?:[^=,]|"  # Type hint: either non-equals/comma chars
+                r"(?<=\[)[^[\]]*"  # or contents within square brackets
+                r"(?=\]))+)"  # looking ahead for closing bracket
+                r"(?:\s*=\s*"  # Optional default value part
+                r"([^,]+))?"  # Default value: anything but comma
+            )
+            match = re.match(param_pattern, line)
+            if not match:
+                continue
+
+            name, type_str = match.group(1), match.group(2).strip()
+
+            # Keep type as string instead of evaluating
+            params.append((name, type_str))
+
+        return params
+
+    return []
+
+
 @classmethod  # type:ignore
 @doc_args(Record.filter.__doc__)
 def filter(cls, *queries, **expressions) -> QuerySet:
@@ -639,8 +744,8 @@ def get_transfer_run(record) -> Run:
     if transform is None:
         search_names = settings.creation.search_names
         settings.creation.search_names = False
-        transform = Transform(
-            uid=uid, name=f"Transfer from `{slug}`", key=key, type="function"
+        transform = Transform(  # type: ignore
+            uid=uid, description=f"Transfer from `{slug}`", key=key, type="function"
         ).save()
         settings.creation.search_names = search_names
     # use the global run context to get the initiated_by_run run id
@@ -655,7 +760,7 @@ def get_transfer_run(record) -> Run:
         transform=transform, initiated_by_run=initiated_by_run
     ).one_or_none()
     if run is None:
-        run = Run(transform=transform, initiated_by_run=initiated_by_run).save()
+        run = Run(transform=transform, initiated_by_run=initiated_by_run).save()  # type: ignore
         run.initiated_by_run = initiated_by_run  # so that it's available in memory
     return run
 
@@ -738,28 +843,40 @@ def save(self, *args, **kwargs) -> Record:
     if pre_existing_record is not None:
         init_self_from_db(self, pre_existing_record)
     else:
-        # save versioned record
-        if isinstance(self, IsVersioned) and self._revises is not None:
-            assert self._revises.is_latest  # noqa: S101
-            revises = self._revises
-            revises.is_latest = False
-            with transaction.atomic():
-                revises._revises = None  # ensure we don't start a recursion
-                revises.save()
-                check_name_change(self)
-                check_key_change(self)
+        check_key_change(self)
+        check_name_change(self)
+        try:
+            # save versioned record in presence of self._revises
+            if isinstance(self, IsVersioned) and self._revises is not None:
+                assert self._revises.is_latest  # noqa: S101
+                revises = self._revises
+                revises.is_latest = False
+                with transaction.atomic():
+                    revises._revises = None  # ensure we don't start a recursion
+                    revises.save()
+                    super(BasicRecord, self).save(*args, **kwargs)  # type: ignore
+                self._revises = None
+            # save unversioned record
+            else:
                 super(BasicRecord, self).save(*args, **kwargs)
-                _store_record_old_name(self)
-                _store_record_old_key(self)
-            self._revises = None
-        # save unversioned record
-        else:
-            check_name_change(self)
-            check_key_change(self)
-            super(BasicRecord, self).save(*args, **kwargs)
-            # update _old_name and _old_key after saving
-            _store_record_old_name(self)
-            _store_record_old_key(self)
+        except IntegrityError as e:
+            error_msg = str(e)
+            # two possible error messages for hash duplication
+            # "duplicate key value violates unique constraint"
+            # "UNIQUE constraint failed"
+            if (
+                "UNIQUE constraint failed" in error_msg
+                or "duplicate key value violates unique constraint" in error_msg
+            ) and "hash" in error_msg:
+                pre_existing_record = self.__class__.get(hash=self.hash)
+                logger.warning(
+                    f"returning {self.__class__.__name__.lower()} with same hash: {pre_existing_record}"
+                )
+                init_self_from_db(self, pre_existing_record)
+            else:
+                raise
+        _store_record_old_name(self)
+        _store_record_old_key(self)
     # perform transfer of many-to-many fields
     # only supported for Artifact and Collection records
     if db is not None and db != "default" and using_key is None:
@@ -778,7 +895,7 @@ def save(self, *args, **kwargs) -> Record:
             self_on_db = copy(self)
             self_on_db._state.db = db
             self_on_db.pk = pk_on_db  # manually set the primary key
-            self_on_db.features = FeatureManager(self_on_db)
+            self_on_db.features = FeatureManager(self_on_db)  # type: ignore
             self.features._add_from(self_on_db, transfer_logs=transfer_logs)
             self.labels.add_from(self_on_db, transfer_logs=transfer_logs)
         for k, v in transfer_logs.items():
@@ -851,7 +968,7 @@ def check_name_change(record: Record):
         # when a feature is renamed
         elif isinstance(record, Feature):
             # only internal features are associated with schemas
-            linked_artifacts = Artifact.filter(_schemas_m2m__features=record).list(
+            linked_artifacts = Artifact.filter(feature_sets__features=record).list(
                 "uid"
             )
             n = len(linked_artifacts)
@@ -915,7 +1032,7 @@ def delete(self) -> None:
             new_latest.is_latest = True
             with transaction.atomic():
                 new_latest.save()
-                super(BasicRecord, self).delete()
+                super(BasicRecord, self).delete()  # type: ignore
             logger.warning(f"new latest version is {new_latest}")
             return None
     super(BasicRecord, self).delete()

lamindb/_run.py

@@ -4,7 +4,7 @@ from lamindb.models import ParamManager, Run, Transform
 
 
 def __init__(run: Run, *args, **kwargs):
-    run.params = ParamManager(run)
+    run.params = ParamManager(run)  # type: ignore
     if len(args) == len(run._meta.concrete_fields):
         super(Run, run).__init__(*args, **kwargs)
         return None
@@ -24,7 +24,7 @@ def __init__(run: Run, *args, **kwargs):
     if transform._state.adding:
         raise ValueError("Please save transform record before creating a run")
 
-    super(Run, run).__init__(
+    super(Run, run).__init__(  # type: ignore
         transform=transform,
         reference=reference,
         initiated_by_run=initiated_by_run,
@@ -56,5 +56,5 @@ def delete(self) -> None:
     super(Run, self).delete()
 
 
-Run.__init__ = __init__
-Run.delete = delete
+Run.__init__ = __init__  # type: ignore
+Run.delete = delete  # type: ignore

lamindb/_save.py

@@ -57,7 +57,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
 
         For a single record, use ``record.save()``:
 
-        >>> transform = ln.Transform(name="My pipeline")
+        >>> transform = ln.Transform(key="My pipeline")
         >>> transform.save()
 
         Update a single existing record:
@@ -133,7 +133,9 @@ def check_and_attempt_upload(
     using_key: str | None = None,
     access_token: str | None = None,
     print_progress: bool = True,
+    **kwargs,
 ) -> Exception | None:
+    # kwargs are propagated to .upload_from in the end
     # if Artifact object is either newly instantiated or replace() was called on
     # a local env it will have a _local_filepath and needs to be uploaded
     if hasattr(artifact, "_local_filepath"):
@@ -143,9 +145,13 @@ def check_and_attempt_upload(
                 using_key,
                 access_token=access_token,
                 print_progress=print_progress,
+                **kwargs,
             )
         except Exception as exception:
             logger.warning(f"could not upload artifact: {artifact}")
+            # clear dangling storages if we were actually uploading or saving
+            if hasattr(artifact, "_to_store") and artifact._to_store:
+                artifact._clear_storagekey = auto_storage_key_from_artifact(artifact)
             return exception
         # copies (if on-disk) or moves the temporary file (if in-memory) to the cache
         if os.getenv("LAMINDB_MULTI_INSTANCE") is None:
@@ -212,19 +218,25 @@ def copy_or_move_to_cache(
 
 # This is also used within Artifact.save()
 def check_and_attempt_clearing(
-    artifact: Artifact, using_key: str | None = None
+    artifact: Artifact,
+    raise_file_not_found_error: bool = True,
+    using_key: str | None = None,
 ) -> Exception | None:
     # this is a clean-up operation after replace() was called
-    # this will only evaluate to True if replace() was called
+    # or if there was an exception during upload
     if hasattr(artifact, "_clear_storagekey"):
         try:
             if artifact._clear_storagekey is not None:
-                delete_storage_using_key(
-                    artifact, artifact._clear_storagekey, using_key=using_key
-                )
-                logger.success(
-                    f"deleted stale object at storage key {artifact._clear_storagekey}"
+                delete_msg = delete_storage_using_key(
+                    artifact,
+                    artifact._clear_storagekey,
+                    raise_file_not_found_error=raise_file_not_found_error,
+                    using_key=using_key,
                 )
+                if delete_msg != "did-not-delete":
+                    logger.success(
+                        f"deleted stale object at storage key {artifact._clear_storagekey}"
+                    )
                 artifact._clear_storagekey = None
         except Exception as exception:
             return exception
@@ -246,11 +258,17 @@ def store_artifacts(
 
     # upload new local artifacts
     for artifact in artifacts:
+        # failure here sets ._clear_storagekey
+        # for cleanup below
         exception = check_and_attempt_upload(artifact, using_key)
         if exception is not None:
             break
         stored_artifacts += [artifact]
-        exception = check_and_attempt_clearing(artifact, using_key)
+        # if check_and_attempt_upload was successfull
+        # then this can have only ._clear_storagekey from .replace
+        exception = check_and_attempt_clearing(
+            artifact, raise_file_not_found_error=True, using_key=using_key
+        )
         if exception is not None:
             logger.warning(f"clean up of {artifact._clear_storagekey} failed")
             break
@@ -261,6 +279,14 @@ def store_artifacts(
             for artifact in artifacts:
                 if artifact not in stored_artifacts:
                     artifact._delete_skip_storage()
+                    # clean up storage after failure in check_and_attempt_upload
+                    exception_clear = check_and_attempt_clearing(
+                        artifact, raise_file_not_found_error=False, using_key=using_key
+                    )
+                    if exception_clear is not None:
+                        logger.warning(
+                            f"clean up of {artifact._clear_storagekey} after the upload error failed"
+                        )
         error_message = prepare_error_message(artifacts, stored_artifacts, exception)
         # this is bad because we're losing the original traceback
         # needs to be refactored - also, the orginal error should be raised
@@ -269,7 +295,7 @@ def store_artifacts(
 
 
 def prepare_error_message(records, stored_artifacts, exception) -> str:
-    if len(records) == 1 or len(stored_artifacts) == 0:
+    if len(stored_artifacts) == 0:
         error_message = (
             "No entries were uploaded or committed"
             " to the database. See error message:\n\n"
@@ -293,8 +319,10 @@ def upload_artifact(
     using_key: str | None = None,
     access_token: str | None = None,
     print_progress: bool = True,
+    **kwargs,
 ) -> tuple[UPath, UPath | None]:
     """Store and add file and its linked entries."""
+    # kwargs are propagated to .upload_from in the end
     # can't currently use  filepath_from_artifact here because it resolves to ._local_filepath
     storage_key = auto_storage_key_from_artifact(artifact)
     storage_path, storage_settings = attempt_accessing_path(
@@ -303,7 +331,10 @@ def upload_artifact(
     if hasattr(artifact, "_to_store") and artifact._to_store:
         logger.save(f"storing artifact '{artifact.uid}' at '{storage_path}'")
         store_file_or_folder(
-            artifact._local_filepath, storage_path, print_progress=print_progress
+            artifact._local_filepath,
+            storage_path,
+            print_progress=print_progress,
+            **kwargs,
         )
 
     if isinstance(storage_path, LocalPathClasses):