lamindb 0.69.0__py3-none-any.whl → 0.69.2__py3-none-any.whl

lamindb/validation/_validator.py

@@ -0,0 +1,221 @@
+from typing import Dict, Iterable, Optional
+
+import pandas as pd
+from lamin_utils import colors, logger
+from lnschema_core.types import FieldAttr
+
+import lamindb as ln
+
+from ._lookup import Lookup
+from ._register import register_artifact, register_labels
+from ._validate import validate_categories_in_df
+
+
+class ValidationError(ValueError):
+    """Validation error."""
+
+    pass
+
+
+class Validator:
+    """Lamin validator.
+
+    Args:
+        df: The DataFrame object to validate.
+        fields: A dictionary mapping column to registry_field.
+            For example:
+            {"cell_type_ontology_id": bt.CellType.ontology_id, "donor_id": ln.ULabel.name}
+        feature_field: The field attribute for the feature column.
+        using: The reference instance containing registries to validate against.
+        verbosity: The verbosity level.
+    """
+
+    def __init__(
+        self,
+        df: pd.DataFrame,
+        fields: Optional[Dict[str, FieldAttr]] = None,
+        feature_field: FieldAttr = ln.Feature.name,
+        using: Optional[str] = None,
+        verbosity: str = "hint",
+        **kwargs,
+    ) -> None:
+        """Initialize the Validator."""
+        self._df = df
+        self._fields = fields or {}
+        self._feature_field = feature_field
+        self._using = using
+        ln.settings.verbosity = verbosity
+        self._artifact = None
+        self._collection = None
+        self._validated = False
+        self._kwargs: Dict = kwargs
+        self.register_features()
+
+    @property
+    def fields(self) -> Dict:
+        """Return the columns fields to validate against."""
+        return self._fields
+
+    def lookup(self, using: Optional[str] = None) -> Lookup:
+        """Lookup features and labels.
+
+        Args:
+            using: The instance where the lookup is performed.
+                if None (default), the lookup is performed on the instance specified in "using" parameter of the Validator.
+                if "public", the lookup is performed on the public reference.
+        """
+        fields = {**{"feature": self._feature_field}, **self.fields}
+        return Lookup(fields=fields, using=using or self._using)
+
+    def register_features(self, validated_only: bool = True) -> None:
+        """Register features records."""
+        missing_columns = set(self.fields.keys()) - set(self._df.columns)
+        if missing_columns:
+            raise ValueError(
+                f"Columns {missing_columns} are not found in the data object!"
+            )
+
+        # Always register features specified as the fields keys
+        register_labels(
+            values=list(self.fields.keys()),
+            field=self._feature_field,
+            feature_name="feature",
+            using=self._using,
+            validated_only=False,
+            kwargs=self._kwargs,
+        )
+
+        # Register the rest of the columns based on validated_only
+        additional_columns = set(self._df.columns) - set(self.fields.keys())
+        if additional_columns:
+            register_labels(
+                values=list(additional_columns),
+                field=self._feature_field,
+                feature_name="feature",
+                using=self._using,
+                validated_only=validated_only,
+                df=self._df,  # Get the Feature type from df
+                kwargs=self._kwargs,
+            )
+
+    def register_labels(self, feature: str, validated_only: bool = True, **kwargs):
+        """Register labels for a feature.
+
+        Args:
+            feature: The name of the feature to register.
+            validated_only: Whether to register only validated labels.
+            **kwargs: Additional keyword arguments.
+        """
+        if feature == "all":
+            self._register_labels_all(validated_only=validated_only, **kwargs)
+        elif feature == "feature":
+            self.register_features(validated_only=validated_only)
+        else:
+            if feature not in self.fields:
+                raise ValueError(f"Feature {feature} is not part of the fields!")
+            register_labels(
+                values=self._df[feature].unique().tolist(),
+                field=self.fields[feature],
+                feature_name=feature,
+                using=self._using,
+                validated_only=validated_only,
+                kwargs=kwargs,
+            )
+
+    def _register_labels_all(self, validated_only: bool = True, **kwargs):
+        """Register labels for all features."""
+        for name in self.fields.keys():
+            logger.info(f"registering labels for '{name}'")
+            self.register_labels(feature=name, validated_only=validated_only, **kwargs)
+
+    def validate(self, **kwargs) -> bool:
+        """Validate variables and categorical observations.
+
+        Returns:
+            Whether the DataFrame is validated.
+        """
+        self._kwargs.update(kwargs)
+        self._validated = validate_categories_in_df(
+            self._df,
+            fields=self.fields,
+            using=self._using,
+            **self._kwargs,
+        )
+        return self._validated
+
+    def register_artifact(self, description: str, **kwargs) -> ln.Artifact:
+        """Register the validated DataFrame and metadata.
+
+        Args:
+            description: Description of the DataFrame object.
+            **kwargs: Object level metadata.
+
+        Returns:
+            A registered artifact record.
+        """
+        self._kwargs.update(kwargs)
+        if not self._validated:
+            raise ValidationError(
+                f"Data object is not validated, please run {colors.yellow('validate()')}!"
+            )
+
+        # Make sure all labels are registered in the current instance
+        verbosity = ln.settings.verbosity
+        try:
+            ln.settings.verbosity = "warning"
+            self.register_labels("all")
+
+            self._artifact = register_artifact(
+                self._df,
+                description=description,
+                fields=self.fields,
+                feature_field=self._feature_field,
+                **self._kwargs,
+            )
+        finally:
+            ln.settings.verbosity = verbosity
+
+        return self._artifact
+
+    def register_collection(
+        self,
+        artifact: ln.Artifact | Iterable[ln.Artifact],
+        name: str,
+        description: Optional[str] = None,
+        reference: Optional[str] = None,
+        reference_type: Optional[str] = None,
+    ) -> ln.Collection:
+        """Register a collection from artifact/artifacts.
+
+        Args:
+            artifact: One or several registered Artifacts.
+            name: Title of the publication.
+            description: Description of the publication.
+            reference: Accession number (e.g. GSE#, E-MTAB#, etc.).
+            reference_type: Source type (e.g. GEO, ArrayExpress, SRA, etc.).
+        """
+        collection = ln.Collection(
+            artifact,
+            name=name,
+            description=description,
+            reference=reference,
+            reference_type=reference_type,
+        )
+        slug = ln.setup.settings.instance.slug
+        if collection._state.adding:
+            collection.save()
+            logger.success(f"registered collection in {colors.italic(slug)}")
+        else:
+            collection.save()
+            logger.warning(f"collection already exists in {colors.italic(slug)}!")
+        if ln.setup.settings.instance.is_remote:
+            logger.print(f"🔗 https://lamin.ai/{slug}/collection/{collection.uid}")
+        self._collection = collection
+        return collection
+
+    def clean_up_failed_runs(self):
+        """Clean up previous failed runs that don't register any outputs."""
+        if ln.run_context.transform is not None:
+            ln.Run.filter(
+                transform=ln.run_context.transform, output_artifacts=None
+            ).exclude(uid=ln.run_context.run.uid).delete()

{lamindb-0.69.0.dist-info → lamindb-0.69.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.69.0
+Version: 0.69.2
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.64.0
-Requires-Dist: lamindb_setup==0.67.0
-Requires-Dist: lamin_utils==0.13.0
-Requires-Dist: lamin_cli==0.10.0
+Requires-Dist: lnschema_core==0.64.1
+Requires-Dist: lamindb_setup==0.68.0
+Requires-Dist: lamin_utils==0.13.1
+Requires-Dist: lamin_cli==0.10.2
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -64,7 +64,7 @@ Provides-Extra: zarr
 - Track data lineage across notebooks & pipelines.
 - Integrate registries for experimental metadata & in-house ontologies.
 - Validate, standardize & annotate.
-- Collaborate across a mesh of distributed LaminDB instances.
+- Collaborate across distributed LaminDB instances.
 
 ## Documentation
 

{lamindb-0.69.0.dist-info → lamindb-0.69.2.dist-info}/RECORD

@@ -1,18 +1,18 @@
-lamindb/__init__.py,sha256=2gJu3U04mfkqWVjXMMvKeVL_QK63rPvqXeboL_GeZQg,2333
-lamindb/_artifact.py,sha256=mpFM8ujnnpmZvBdB_BOpp0YclqD9oNj9y2qrDUDYjyo,36430
-lamindb/_collection.py,sha256=esjFYaVPIz7E5J-dk3FppAOZsimszd4m7MYGxErmrk4,18097
-lamindb/_feature.py,sha256=B_vjiH7wOW0mugZOmIHYKkAe20E1Ca9PnMppAW9KJU4,6677
+lamindb/__init__.py,sha256=hJStNsXJq-qclYj7tDUz2t-4j5sDhkZdBen5URQ1_dA,2051
+lamindb/_artifact.py,sha256=3H8hemGysZLlyHkb02MEXCie1FluQ60LdGIBXOv13uc,35999
+lamindb/_collection.py,sha256=03CQ0u8eCY_dx31pIT5ZMZsmxbbj6J5dJ9zUqJLrDGY,18427
+lamindb/_feature.py,sha256=ahRv87q1tcRLQ0UM5FA3KtcMQvIjW__fZq1yAdRAV7s,6728
 lamindb/_feature_set.py,sha256=G_Ss6mKh4D0Eji-xSfLRbKVFXwgUE82YOqIUmkV0CAA,8767
 lamindb/_filter.py,sha256=_PjyQWQBR3ohDAvJbR3hMvZ-2p2GvzFxLfKGC-gPnHI,1320
-lamindb/_finish.py,sha256=2YdHcHXZeu5414jIbk9id-u5_m9W73hfqzLH-AwYK1Y,1269
+lamindb/_finish.py,sha256=it-fSpSmMW9ybdsylBV5Lbugh6iXRGWgIiSLwPaow_8,8590
 lamindb/_from_values.py,sha256=Ei11ml77Q1xubVekt2C4-mbox2-qnC7kP18B-LhCdSc,11886
 lamindb/_is_versioned.py,sha256=DXp5t-1DwErpqqMc9eb08kpQPCHOC2fNzaozMoBunR4,1337
 lamindb/_parents.py,sha256=pTDsW8HjQ_txFbPKrBU0WjjtCNH6sx2LASUuGWpJuYE,14742
 lamindb/_query_manager.py,sha256=lyYMEsstUQlns2H01oZXN5Ly0X6ug2VOPebyu9fHn4s,4008
-lamindb/_query_set.py,sha256=JXw43IoSW0QVmarOmRqZGBNn4aFUYetwWEAJcoG_g6Q,11235
+lamindb/_query_set.py,sha256=OXL5meaGoWHV5aPhT-LYUboPHFB0i1BPWfmvKTSeYF4,11306
 lamindb/_registry.py,sha256=vEsjn33BV2vxlanE3fyvDiy7AJoq7RKqEn_Sspo4_Dc,19232
-lamindb/_run.py,sha256=O5TcVPZqgdUvk4nBmI_HXzvWOG_Zr3B_HMDaw228w-4,1719
-lamindb/_save.py,sha256=2V3u0F2FFJZBlaBtViBYLIPDIVlwY-uXlhPDfA-wCMo,11102
+lamindb/_run.py,sha256=CvH6cAFUb83o38iOdpBsktF3JGAwmuZrDZ4p4wvUr0g,1853
+lamindb/_save.py,sha256=uIzHfNulzn7rpSKyAvUHT1OuN294OWFGC04gLmwrScY,11452
 lamindb/_storage.py,sha256=VW8xq3VRv58-ciholvOdlcgvp_OIlLxx5GxLt-e2Irs,614
 lamindb/_transform.py,sha256=oZq-7MgyCs4m6Bj901HwDlbvF0JuvTpe3RxN0Zb8PgE,3515
 lamindb/_ulabel.py,sha256=euXsDPD7wC99oopLXVkT-vq7f3E6-zP4Z4akI-yh0aM,1913
@@ -20,11 +20,11 @@ lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_validate.py,sha256=w7lrUGTWldpvwaRiXBRrjfU_ZRidA7CooOu_r5MbocY,14569
 lamindb/_view.py,sha256=yFMu4vnt0YqvN1q11boAkwigxCH1gdliDUSbzh3IuDw,2175
 lamindb/core/__init__.py,sha256=RYNsg2foVZRawpCW2J5J82vHZt6ub_Tze8wiDMxXCH8,988
-lamindb/core/_data.py,sha256=D89NN22Py832wT53U5neFNpvft_U4_WQS4XcFNQ0fVM,17214
-lamindb/core/_feature_manager.py,sha256=a1HXGmMgSRWEsE7nmlMMKYZzDo8oFAmieOM6HGa-yyU,13901
+lamindb/core/_data.py,sha256=Q8w1I8pXXOaLVIxfjWBkLV6GGnzaQxCXamu9tplFgsA,17287
+lamindb/core/_feature_manager.py,sha256=II0nuxtjOdEtU_9a7eB18_Clw9d1n5k1JOqk_vHisRw,13940
 lamindb/core/_label_manager.py,sha256=zrWDSd2AkR6fKsGDxLSWqHC9fz9BcGlavPZEh92Wzjg,9063
 lamindb/core/_mapped_collection.py,sha256=e4P3AoykIMjD4_88BWbISWvKyWWTklwHl-_WLa72ZG4,16841
-lamindb/core/_run_context.py,sha256=bwMHgO01oA4DxWTFqj4VDNBPS7Ti89Heuh1vHPv7HjU,17063
+lamindb/core/_run_context.py,sha256=EK0lFJWx32NY2FdqFR1YozR9zioC-BjA394nPu-KwLQ,17510
 lamindb/core/_settings.py,sha256=kHL5e20dWKSbf7mJOAddvS7SQBrr1D0ZTeG_5sj5RpY,5735
 lamindb/core/_sync_git.py,sha256=Bn_ofx2ynaw6etmskgEUNW8n7LDJs-7r2aB41BgCvdA,3928
 lamindb/core/_track_environment.py,sha256=QjHWbyl2u8J4hbJG8Q_ToFaZIgS-H15Ej6syJgk-dvY,662
@@ -39,13 +39,19 @@ lamindb/core/datasets/_core.py,sha256=Y1UP_gPN2w6-QijaqmeKV57luYXYb5d2G-bmuSobS1
 lamindb/core/datasets/_fake.py,sha256=S8mNho-oSh1M9x9oOSsUBLLHmBAegsOLlFk6LnF81EA,942
 lamindb/core/storage/__init__.py,sha256=9alBNtyH59VnoWJS-IdjLwFKlK-kgeCGl6jXk0_wGeQ,369
 lamindb/core/storage/_anndata_sizes.py,sha256=0XVzA6AQeVGPaGPrhGusKyxFgFjeo3qSN29hxb8D5E8,993
-lamindb/core/storage/_backed_access.py,sha256=MD58DbKMen-62I7eloaCSUFYpU2zTERGbJuPWus2qsE,22871
+lamindb/core/storage/_backed_access.py,sha256=DUJIDjkGkemjmKLD05blndP_rO5DpUD0EZdowos46HQ,24361
 lamindb/core/storage/_zarr.py,sha256=bMQSCsTOCtQy4Yo3KwCVpbUkKdWRApN9FM1rM-d2_G0,2839
-lamindb/core/storage/file.py,sha256=JmDnux5aNL2L30Tkyb4-uRgavlFfEJrQjt9X8957lYY,7321
+lamindb/core/storage/file.py,sha256=WTeC4ENn_O6HEoinmTviB89W81UrJT3bSGtnpqPpIyE,7242
 lamindb/core/storage/object.py,sha256=MPUb2M8Fleq2j9x1Ryqr3BETmvsDKyf11Ifvbxd3NpA,1097
 lamindb/setup/__init__.py,sha256=OwZpZzPDv5lPPGXZP7-zK6UdO4FHvvuBh439yZvIp3A,410
 lamindb/setup/core/__init__.py,sha256=LqIIvJNcONxkqjbnP6CUaP4d45Lbd6TSMAcXFp4C7_8,231
-lamindb-0.69.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.69.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.69.0.dist-info/METADATA,sha256=3rjzCNybJaQrlJZlVwSuHCtyIwu2HRSr5SIlXi4Z_S8,2866
-lamindb-0.69.0.dist-info/RECORD,,
+lamindb/validation/__init__.py,sha256=AuonqVEhyYDXAoRqXnM9JweTUnYfAoExza8A5mQuM7Q,347
+lamindb/validation/_anndata_validator.py,sha256=lFCVLE4F4VN-9DTEwY9RUqSw8I2C6eTPYvXotGdKgvU,3782
+lamindb/validation/_lookup.py,sha256=HIGwk85e-c8yaVg4NkcvBdW4LIhnxwRI02km8uYOiFY,1545
+lamindb/validation/_register.py,sha256=UKsNVwXZhBl-spheZX1nkugjLF8g1yANT2vumcyzx6Y,9765
+lamindb/validation/_validate.py,sha256=FPQ4e_qDcP3tlKsYOVyo7-yb8nIbKyzoZHwgMbJJog0,4588
+lamindb/validation/_validator.py,sha256=6vzOfKIPQdA0pWwtXlRJWvjgLIjpivkBeLtgD6QODvY,7861
+lamindb-0.69.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.69.2.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.69.2.dist-info/METADATA,sha256=ly2Nwd236G0yxp4sX3DStxyzFFzqSv7sJuccmnc142Y,2856
+lamindb-0.69.2.dist-info/RECORD,,