lamindb 1.9.1__py3-none-any.whl → 1.10.1__py3-none-any.whl

lamindb/integrations/_croissant.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import lamindb as ln
+
+
+def curate_from_croissant(
+    croissant_data: str | Path | dict[str, Any],
+    run: ln.Run | None = None,
+) -> ln.Artifact | ln.Collection:
+    """Create annotated artifacts from a CroissantML file.
+
+    Returns a collection if multiple files are found in `croissant_data`, otherwise a single artifact.
+
+    Args:
+        croissant_data: Path to CroissantML JSON file or dictionary.
+
+    Example:
+
+        ::
+
+            artifact = ln.integrations.curate_from_croissant("dataset_metadata.json")
+    """
+    import lamindb as ln
+
+    # Load CroissantML data
+    if isinstance(croissant_data, (str, Path)):
+        if not Path(croissant_data).exists():
+            raise FileNotFoundError(f"File not found: {croissant_data}")
+        with open(croissant_data, encoding="utf-8") as f:
+            data = json.load(f)
+    elif isinstance(croissant_data, dict):
+        data = croissant_data
+    else:
+        raise ValueError(
+            "croissant_data must be a file path, JSON string, or dictionary"
+        )
+
+    # Validate basic structure
+    if data.get("@type") != "Dataset":
+        raise ValueError("CroissantML @type must be 'Dataset'")
+
+    if "name" not in data:
+        raise ValueError("CroissantML must have a 'name' field")
+
+    # Extract basic metadata
+    dataset_name = data["name"]
+    description = data.get("description", "")
+    version = data.get("version", "1.0")
+    license_info = data.get("license", "")
+    project_name = data.get("cr:projectName", "")
+
+    # Create license feature and label if license info exists
+    license_label = None
+    if license_info:
+        license_label_type = ln.ULabel.filter(name="License", is_type=True).first()
+        if not license_label_type:
+            license_label_type = ln.ULabel(name="License", is_type=True).save()
+        license_label = ln.ULabel.filter(name=license_info).first()
+        if not license_label:
+            license_label = ln.ULabel(
+                name=license_info,
+                description="Dataset license",
+                type=license_label_type,
+            ).save()
+    project_label = None
+    if project_name:
+        project_label = ln.Project.filter(name=project_name).first()
+        if not project_label:
+            project_label = ln.Project(name=project_name).save()
+
+    # Extract file distributions
+    artifacts = []
+    file_distributions = data.get("distribution", [])
+    if not file_distributions:
+        raise ValueError("No file distributions found in croissant data")
+    for dist in file_distributions:
+        file_id = dist.get("@id", "")
+        if Path(file_id).exists():
+            file_path = file_id
+        else:
+            content_url = dist.get("contentUrl", "")
+            file_path = content_url or data.get("url", "")
+        if not file_path:
+            raise ValueError(
+                f"No valid file path found in croissant distribution: {dist}"
+            )
+        if len(file_distributions) == 1:
+            artifact_description = f"{dataset_name}"
+            if file_id != dataset_name:
+                artifact_description += f" ({file_id})"
+            artifact_description += f" - {description}"
+        else:
+            artifact_description = f"{file_id}"
+        artifact = ln.Artifact(  # type: ignore
+            file_path,
+            description=artifact_description,
+            version=version,
+            kind="dataset",
+            run=run,
+        ).save()
+        if license_label:
+            artifact.ulabels.add(license_label)
+        if project_label:
+            artifact.projects.add(project_label)
+        artifacts.append(artifact)
+
+    if len(artifacts) == 1:
+        return artifacts[0]
+    else:
+        collection = ln.Collection(  # type: ignore
+            artifacts, key=dataset_name, description=description, version=version
+        ).save()
+        if license_label:
+            collection.ulabels.add(license_label)
+        if project_label:
+            collection.projects.add(project_label)
+        return collection

lamindb/integrations/_vitessce.py

@@ -28,21 +28,17 @@ def save_vitessce_config(
     If the `VitessceConfig` object references multiple artifacts, automatically
     creates a `Collection` and displays the "Vitessce button" next to it.
 
+    The `VitessceConfig` artifact has `.suffix = ".vitessce.json"` and `.kind = "__lamindb_config__"`,
+    which is by default hidden on the hub UI.
+
     Guide: :doc:`docs:vitessce`.
 
     Args:
         vitessce_config: A `VitessceConfig` object.
-        key: A key for the `VitessceConfig` object. Is used as `key` for a
-            `Collection` in case the `VitessceConfig` object references
-            multiple artifacts.
-        description: A description for the `VitessceConfig` object.
-
-    .. versionchanged:: 0.76.12
-        Now assumes `vitessce-python >= 3.4.0`, which allows passing artifacts within `VitessceConfig`.
-    .. versionchanged:: 0.75.1
-        Now displays the "Vitessce button" on the hub next to the dataset. It additionally keeps displaying it next to the configuration file.
-    .. versionchanged:: 0.70.2
-        No longer saves the dataset. It only saves the `VitessceConfig` object.
+        key: A `key` for the `VitessceConfig` artifact.
+        description: A `description` for the `VitessceConfig` aritifact. Is additionally
+            used as `key` for a `Collection` in case the `VitessceConfig` object
+            references multiple artifacts.
     """
     # can only import here because vitessce is not a dependency
     from vitessce import VitessceConfig
@@ -73,6 +69,8 @@ def save_vitessce_config(
     if len(dataset_artifacts) > 1:
         # if we have more datasets, we should create a collection
         # and attach an action to the collection
+        # consicious use of description for key, see here
+        # https://github.com/laminlabs/lamindb/pull/2997
         collection = Collection(dataset_artifacts, key=description).save()
 
     # create a JSON export
@@ -80,7 +78,11 @@ def save_vitessce_config(
     with open(config_file_local_path, "w") as file:
         json.dump(vc_dict, file)
     vitessce_config_artifact = Artifact(
-        config_file_local_path, key=key, description=description, run=run
+        config_file_local_path,
+        key=key,
+        description=description,
+        run=run,
+        kind="__lamindb_config__",
     ).save()
     slug = ln_setup.settings.instance.slug
     logger.important(

lamindb/migrations/0116_remove_artifact_unique_artifact_storage_key_hash_and_more.py

@@ -0,0 +1,51 @@
+# Generated by Django 5.2 on 2025-07-26 15:55
+
+from django.db import migrations, models
+
+import lamindb.base.fields
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0115_alter_space_uid"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="artifact",
+            name="unique_artifact_storage_key_hash",
+        ),
+        migrations.AlterField(
+            model_name="record",
+            name="description",
+            field=lamindb.base.fields.CharField(
+                blank=True, db_index=True, default=None, max_length=255, null=True
+            ),
+        ),
+        migrations.AlterField(
+            model_name="reference",
+            name="text",
+            field=lamindb.base.fields.TextField(
+                blank=True, db_index=True, default=None, null=True
+            ),
+        ),
+        migrations.AlterField(
+            model_name="reference",
+            name="url",
+            field=lamindb.base.fields.URLField(blank=True, db_index=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="run",
+            name="name",
+            field=lamindb.base.fields.CharField(
+                blank=True, db_index=True, default=None, max_length=150, null=True
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="artifact",
+            constraint=models.UniqueConstraint(
+                fields=("storage", "key", "hash"),
+                name="unique_artifact_storage_key_hash",
+            ),
+        ),
+    ]

lamindb/migrations/0117_fix_artifact_storage_hash_unique_constraints.py

@@ -0,0 +1,32 @@
+# Generated by Django 5.2 on 2025-07-26 18:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0116_remove_artifact_unique_artifact_storage_key_hash_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="artifact",
+            name="unique_artifact_storage_key_hash",
+        ),
+        migrations.AddConstraint(
+            model_name="artifact",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("key__isnull", False)),
+                fields=("storage", "key", "hash"),
+                name="unique_artifact_storage_key_hash_not_null",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="artifact",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("key__isnull", True)),
+                fields=("storage", "hash"),
+                name="unique_artifact_storage_hash_null_key",
+            ),
+        ),
+    ]

lamindb/migrations/{0115_squashed.py → 0117_squashed.py}

@@ -1,4 +1,4 @@
-# Generated by Django 5.2 on 2025-07-06 09:12
+# Generated by Django 5.2 on 2025-07-26 18:58
 
 import django.core.validators
 import django.db.models.deletion
@@ -137,6 +137,8 @@ class Migration(migrations.Migration):
         ("lamindb", "0113_lower_case_branch_and_space_names"),
         ("lamindb", "0114_alter_run__status_code"),
         ("lamindb", "0115_alter_space_uid"),
+        ("lamindb", "0116_remove_artifact_unique_artifact_storage_key_hash_and_more"),
+        ("lamindb", "0117_fix_artifact_storage_hash_unique_constraints"),
     ]
 
     dependencies = []  # type: ignore
@@ -1477,7 +1479,11 @@ class Migration(migrations.Migration):
                 (
                     "description",
                     lamindb.base.fields.CharField(
-                        blank=True, default=None, max_length=255, null=True
+                        blank=True,
+                        db_index=True,
+                        default=None,
+                        max_length=255,
+                        null=True,
                     ),
                 ),
                 (
@@ -1825,7 +1831,10 @@ class Migration(migrations.Migration):
                         null=True,
                     ),
                 ),
-                ("url", lamindb.base.fields.URLField(blank=True, null=True)),
+                (
+                    "url",
+                    lamindb.base.fields.URLField(blank=True, db_index=True, null=True),
+                ),
                 (
                     "pubmed_id",
                     lamindb.base.fields.BigIntegerField(
@@ -1860,7 +1869,9 @@ class Migration(migrations.Migration):
                 ),
                 (
                     "text",
-                    lamindb.base.fields.TextField(blank=True, default=None, null=True),
+                    lamindb.base.fields.TextField(
+                        blank=True, db_index=True, default=None, null=True
+                    ),
                 ),
                 (
                     "date",
@@ -1989,7 +2000,11 @@ class Migration(migrations.Migration):
                 (
                     "name",
                     lamindb.base.fields.CharField(
-                        blank=True, default=None, max_length=150, null=True
+                        blank=True,
+                        db_index=True,
+                        default=None,
+                        max_length=150,
+                        null=True,
                     ),
                 ),
                 (
@@ -4472,7 +4487,15 @@ class Migration(migrations.Migration):
             constraint=models.UniqueConstraint(
                 condition=models.Q(("key__isnull", False)),
                 fields=("storage", "key", "hash"),
-                name="unique_artifact_storage_key_hash",
+                name="unique_artifact_storage_key_hash_not_null",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="artifact",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("key__isnull", True)),
+                fields=("storage", "hash"),
+                name="unique_artifact_storage_hash_null_key",
             ),
         ),
     ]

lamindb/models/_describe.py

@@ -179,7 +179,7 @@ def describe_artifact_general(
     two_column_items.append(Text.assemble(("branch: ", "dim"), branch_name))
     # actually not name field here, but handle
     created_by_handle = (
-        foreign_key_data["branch"]["name"]
+        foreign_key_data["created_by"]["name"]
         if foreign_key_data
         else self.created_by.handle
     )
@@ -234,3 +234,109 @@ def describe_artifact_general(
         )
     )
     return tree
+
+
+def describe_collection_general(
+    self: Collection,
+    tree: Tree | None = None,
+    foreign_key_data: dict[str, dict[str, int | str]] | None = None,
+) -> Tree:
+    if tree is None:
+        tree = describe_header(self)
+
+    # add general information (order is the same as in API docs)
+    general = tree.add(Text("General", style="bold bright_cyan"))
+
+    if self.key:
+        general.add(Text.assemble(("key: ", "dim"), (f"{self.key}", "cyan3")))
+    if self.description:
+        general.add(
+            Text.assemble(
+                ("description: ", "dim"),
+                f"{self.description}",
+            )
+        )
+
+    # Two column items (short content)
+    two_column_items = []
+
+    two_column_items.append(Text.assemble(("uid: ", "dim"), f"{self.uid}"))
+
+    transform_name = (
+        foreign_key_data["transform"]["name"]
+        if foreign_key_data and "transform" in foreign_key_data
+        else self.transform.name
+        if self.transform
+        else None
+    )
+    if transform_name:
+        two_column_items.append(
+            Text.assemble(
+                ("transform: ", "dim"),
+                (f"{transform_name}", "cyan3"),
+            )
+        )
+
+    space_name = (
+        foreign_key_data["space"]["name"]
+        if foreign_key_data and "space" in foreign_key_data
+        else self.space.name
+        if self.space
+        else None
+    )
+    if space_name:
+        two_column_items.append(Text.assemble(("space: ", "dim"), space_name))
+
+    branch_name = (
+        foreign_key_data["branch"]["name"]
+        if foreign_key_data and "branch" in foreign_key_data
+        else self.branch.name
+        if self.branch
+        else None
+    )
+    if branch_name:
+        two_column_items.append(Text.assemble(("branch: ", "dim"), branch_name))
+
+    created_by_handle = (
+        foreign_key_data["created_by"]["name"]
+        if foreign_key_data and "created_by" in foreign_key_data
+        else self.created_by.handle
+        if self.created_by
+        else None
+    )
+    if created_by_handle:
+        two_column_items.append(
+            Text.assemble(
+                ("created_by: ", "dim"),
+                (created_by_handle),
+            )
+        )
+
+    if self.created_at:
+        two_column_items.append(
+            Text.assemble(("created_at: ", "dim"), highlight_time(str(self.created_at)))
+        )
+
+    if self.version:
+        two_column_items.append(Text.assemble(("version: ", "dim"), f"{self.version}"))
+
+    # Add two-column items in pairs
+    for i in range(0, len(two_column_items), 2):
+        if i + 1 < len(two_column_items):
+            # Two items side by side
+            left_item = two_column_items[i]
+            right_item = two_column_items[i + 1]
+
+            # Create padded version by calculating the plain text length
+            left_plain_text = (
+                left_item.plain if hasattr(left_item, "plain") else str(left_item)
+            )
+            padding_needed = max(0, 45 - len(left_plain_text))
+            padding = " " * padding_needed
+
+            general.add(Text.assemble(left_item, padding, right_item))
+        else:
+            # Single item (odd number)
+            general.add(two_column_items[i])
+
+    return tree

lamindb/models/_django.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.db import connection
@@ -13,8 +13,7 @@ from ._relations import dict_related_model_to_related_name, get_schema_modules
 from .schema import Schema
 
 if TYPE_CHECKING:
-    from .artifact import Artifact
-    from .sqlrecord import SQLRecord
+    from .artifact import Artifact, Collection
 
 
 def patch_many_to_many_descriptor() -> None:
@@ -32,7 +31,8 @@ def patch_many_to_many_descriptor() -> None:
     def patched_get(self, instance, cls=None):
         if instance is not None and instance.pk is None:
             raise ValueError(
-                f"You are trying to access the many-to-many relationships of an unsaved {instance.__class__.__name__} object. Please save it first using '.save()'."
+                f"You are trying to access the many-to-many relationships of an unsaved {instance.__class__.__name__} object. "
+                f"Please save it first using '.save()'."
             )
 
         manager = original_get(self, instance, cls)
@@ -77,12 +77,12 @@ def get_related_model(model, field_name):
 
 
 def get_artifact_with_related(
-    artifact: SQLRecord,
+    artifact: Artifact,
     include_fk: bool = False,
     include_m2m: bool = False,
     include_feature_link: bool = False,
     include_schema: bool = False,
-) -> dict:
+) -> dict[str, Any]:
     """Fetch an artifact with its related data."""
     from ._label_manager import EXCLUDE_LABELS
     from .can_curate import get_name_field
@@ -234,6 +234,63 @@ def get_artifact_with_related(
     }
 
 
+def get_collection_with_related(
+    collection: Collection,
+    include_fk: bool = False,
+) -> dict[str, Any]:
+    """Fetch a collection with its related data."""
+    from .can_curate import get_name_field
+
+    model = collection.__class__
+    schema_modules = get_schema_modules(collection._state.db)
+
+    foreign_key_fields = [
+        f.name
+        for f in model._meta.fields
+        if f.is_relation and f.related_model.__get_module_name__() in schema_modules
+    ]
+
+    # Clear previous queries
+    connection.queries_log.clear()
+
+    annotations = {}
+
+    if include_fk:
+        for fk in foreign_key_fields:
+            name_field = get_name_field(get_related_model(model, fk))
+            if fk == "run":
+                annotations[f"fkfield_{fk}"] = JSONObject(
+                    id=F(f"{fk}__id"),
+                    name=F(f"{fk}__{name_field}"),
+                    transform_key=F(f"{fk}__transform__key"),
+                )
+            else:
+                annotations[f"fkfield_{fk}"] = JSONObject(
+                    id=F(f"{fk}__id"), name=F(f"{fk}__{name_field}")
+                )
+
+    collection_meta = (
+        model.objects.using(collection._state.db)
+        .filter(uid=collection.uid)
+        .annotate(**annotations)
+        .values(*["id", "uid"], *annotations.keys())
+        .first()
+    )
+
+    if not collection_meta:
+        return None
+
+    related_data: dict = {"fk": {}}
+    for k, v in collection_meta.items():
+        if k.startswith("fkfield_") and v is not None:
+            related_data["fk"][k[8:]] = v
+
+    return {
+        **{name: collection_meta[name] for name in ["id", "uid"]},
+        "related_data": related_data,
+    }
+
+
 def get_schema_m2m_relations(artifact: Artifact, slot_schema: dict, limit: int = 20):
     """Fetch all many-to-many relationships for given feature sets."""
     from .can_curate import get_name_field

lamindb/models/_feature_manager.py

@@ -969,7 +969,6 @@ class FeatureManager:
                         }
                     else:
                         result = parse_dtype(feature.dtype)[0]
-                        print(result["field"])
                     validated = result["registry"].validate(  # type: ignore
                         values, field=result["field"], mute=True
                     )