lamindb 0.72.1__py3-none-any.whl → 0.73.0__py3-none-any.whl

lamindb/__init__.py

@@ -41,7 +41,7 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.72.1"
+__version__ = "0.73.0"
 
 import os as _os
 

lamindb/_annotate.py

@@ -22,15 +22,13 @@ class AnnotateLookup:
 
     def __init__(
         self,
-        categorials: dict[str, FieldAttr],
+        categoricals: dict[str, FieldAttr],
         slots: dict[str, FieldAttr] = None,
         using: str | None = None,
     ) -> None:
         if slots is None:
             slots = {}
-        if slots is None:
-            slots = {}
-        self._fields = {**categorials, **slots}
+        self._fields = {**categoricals, **slots}
         self._using = None if using == "default" else using
         self._using_name = self._using or ln_setup.settings.instance.slug
         debug_message = f"Lookup objects from the " f"{colors.italic(self._using_name)}"
@@ -73,7 +71,7 @@ class AnnotateLookup:
                 "Example:\n    → categories = validator.lookup().cell_type\n"
                 "    → categories.alveolar_type_1_fibroblast_cell"
             )
-        else:
+        else:  # pragma: no cover
             return colors.warning("No fields are found!")
 
 
@@ -132,7 +130,7 @@ class DataFrameAnnotator:
                 if "public", the lookup is performed on the public reference.
         """
         return AnnotateLookup(
-            categorials=self._fields,
+            categoricals=self._fields,
             slots={"columns": self._columns_field},
             using=using or self._using,
         )
@@ -305,10 +303,10 @@ class DataFrameAnnotator:
         slug = ln_setup.settings.instance.slug
         if collection._state.adding:
             collection.save()
-        else:
+        else:  # pragma: no cover
             collection.save()
             logger.warning(f"collection already exists in {colors.italic(slug)}!")
-        if ln_setup.settings.instance.is_remote:
+        if ln_setup.settings.instance.is_remote:  # pragma: no cover
             logger.print(f"go to https://lamin.ai/{slug}/collection/{collection.uid}")
         self._collection = collection
         return collection
@@ -363,7 +361,7 @@ class AnnDataAnnotator(DataFrameAnnotator):
             )
         if isinstance(data, ad.AnnData):
             self._adata = data
-        else:
+        else:  # pragma: no cover
             from lamindb.core.storage._backed_access import backed_access
 
             self._adata = backed_access(upath.create_path(data))
@@ -399,7 +397,7 @@ class AnnDataAnnotator(DataFrameAnnotator):
                 if "public", the lookup is performed on the public reference.
         """
         return AnnotateLookup(
-            categorials=self._obs_fields,
+            categoricals=self._obs_fields,
             slots={"columns": self._columns_field, "var_index": self._var_field},
             using=using or self._using,
         )
@@ -466,7 +464,9 @@ class AnnDataAnnotator(DataFrameAnnotator):
             A saved artifact record.
         """
         if not self._validated:
-            raise ValidationError("Please run `validate()` first!")
+            raise ValidationError(
+                f"Data object is not validated, please run {colors.yellow('validate()')}!"
+            )
 
         self._artifact = save_artifact(
             self._data,
@@ -489,6 +489,7 @@ class MuDataAnnotator:
             For example:
             ``{"modality_1": bt.Gene.ensembl_gene_id, "modality_2": ln.CellMarker.name}``
         categoricals: A dictionary mapping ``.obs.columns`` to a registry field.
+            Use modality keys to specify categoricals for MuData slots such as `"rna:cell_type": bt.CellType.name"`.
         using: A reference LaminDB instance.
         verbosity: The verbosity level.
         organism: The organism name.
@@ -593,7 +594,7 @@ class MuDataAnnotator:
                 if "public", the lookup is performed on the public reference.
         """
         return AnnotateLookup(
-            categorials=self._obs_fields,
+            categoricals=self._obs_fields,
             slots={
                 **self._obs_fields,
                 **{f"{k}_var_index": v for k, v in self._var_fields.items()},
@@ -988,11 +989,15 @@ def save_artifact(
     )
 
     if artifact.accessor == "DataFrame":
-        artifact.features.add_from_df(field=columns_field, **feature_kwargs)
+        artifact.features._add_set_from_df(field=columns_field, **feature_kwargs)
     elif artifact.accessor == "AnnData":
-        artifact.features.add_from_anndata(var_field=columns_field, **feature_kwargs)
+        artifact.features._add_set_from_anndata(
+            var_field=columns_field, **feature_kwargs
+        )
     elif artifact.accessor == "MuData":
-        artifact.features.add_from_mudata(var_fields=columns_field, **feature_kwargs)
+        artifact.features._add_set_from_mudata(
+            var_fields=columns_field, **feature_kwargs
+        )
     else:
         raise NotImplementedError
 
@@ -1016,7 +1021,7 @@ def save_artifact(
         _add_labels(data, artifact, fields)
 
     slug = ln_setup.settings.instance.slug
-    if ln_setup.settings.instance.is_remote:
+    if ln_setup.settings.instance.is_remote:  # pragma: no cover
         logger.important(f"go to https://lamin.ai/{slug}/artifact/{artifact.uid}")
     return artifact
 
@@ -1124,6 +1129,8 @@ def log_saved_labels(
     validated_only: bool = True,
 ) -> None:
     """Log the saved labels."""
+    from ._from_values import _print_values
+
     model_field = colors.italic(model_field)
     for k, labels in labels_saved.items():
         if not labels:
@@ -1151,7 +1158,7 @@ def log_saved_labels(
             # labels from a public ontology or a different instance to the present instance
             s = "s" if len(labels) > 1 else ""
             logger.success(
-                f"added {len(labels)} record{s} {k}with {model_field} for {colors.italic(key)}: {labels}"
+                f"added {len(labels)} record{s} {k}with {model_field} for {colors.italic(key)}: {_print_values(labels)}"
             )
 
 
@@ -1204,7 +1211,7 @@ def update_registry_from_using_instance(
     return labels_saved, not_saved
 
 
-def _save_organism(name: str):
+def _save_organism(name: str):  # pragma: no cover
     """Save an organism record."""
     import bionty as bt
 

lamindb/_artifact.py

@@ -23,13 +23,13 @@ from lamindb_setup.core.upath import (
     get_stat_dir_cloud,
     get_stat_file_cloud,
 )
-from lnschema_core import Artifact, Run, Storage
+from lnschema_core.models import Artifact, FeatureManager, Run, Storage
 from lnschema_core.types import (
     VisibilityChoice,
 )
 
 from lamindb._utils import attach_func_to_class_method
-from lamindb.core._data import Data, _track_run_input
+from lamindb.core._data import HasFeatures, _track_run_input
 from lamindb.core._settings import settings
 from lamindb.core.storage import (
     LocalPathClasses,
@@ -186,8 +186,6 @@ def process_data(
 
 def get_stat_or_artifact(
     path: UPath,
-    suffix: str,
-    memory_rep: Any | None = None,
     check_hash: bool = True,
     using_key: str | None = None,
 ) -> tuple[int, str | None, str | None, int | None] | Artifact:
@@ -261,7 +259,7 @@ def get_stat_or_artifact(
                     f"You're trying to re-create this artifact in trash: {result[0]}"
                     "Either permanently delete it with `artifact.delete(permanent=True)` or restore it with `artifact.restore()`"
                 )
-            logger.warning(f"returning existing artifact with same hash: {result[0]}")
+            logger.important(f"returning existing artifact with same hash: {result[0]}")
             return result[0]
     else:
         return size, hash, hash_type, n_objects
@@ -338,8 +336,6 @@ def get_artifact_kwargs_from_data(
     )
     stat_or_artifact = get_stat_or_artifact(
         path=path,
-        suffix=suffix,
-        memory_rep=memory_rep,
         using_key=using_key,
     )
     if isinstance(stat_or_artifact, Artifact):
@@ -509,7 +505,7 @@ def _check_accessor_artifact(data: Any, accessor: str | None = None):
     return accessor
 
 
-def update_attributes(data: Data, attributes: Mapping[str, str]):
+def update_attributes(data: HasFeatures, attributes: Mapping[str, str]):
     for key, value in attributes.items():
         if getattr(data, key) != value:
             logger.warning(f"updated {key} from {getattr(data, key)} to {value}")
@@ -517,6 +513,7 @@ def update_attributes(data: Data, attributes: Mapping[str, str]):
 
 
 def __init__(artifact: Artifact, *args, **kwargs):
+    artifact.features = FeatureManager(artifact)
     # Below checks for the Django-internal call in from_db()
     # it'd be better if we could avoid this, but not being able to create a Artifact
     # from data with the default constructor renders the central class of the API
@@ -1006,7 +1003,7 @@ def delete(
         # we don't yet have logic to bring back the deleted metadata record
         # in case storage deletion fails - this is important for ACID down the road
         if delete_in_storage:
-            delete_msg = delete_storage(path)
+            delete_msg = delete_storage(path, raise_file_not_found_error=False)
             if delete_msg != "did-not-delete":
                 logger.success(f"deleted {colors.yellow(f'{path}')}")
 
@@ -1018,6 +1015,7 @@ def _delete_skip_storage(artifact, *args, **kwargs) -> None:
 # docstring handled through attach_func_to_class_method
 def save(self, upload: bool | None = None, **kwargs) -> None:
     state_was_adding = self._state.adding
+    print_progress = kwargs.pop("print_progress", True)
     access_token = kwargs.pop("access_token", None)
     local_path = None
     if upload and setup_settings.instance.keep_artifacts_local:
@@ -1038,7 +1036,9 @@ def save(self, upload: bool | None = None, **kwargs) -> None:
     using_key = None
     if "using" in kwargs:
         using_key = kwargs["using"]
-    exception = check_and_attempt_upload(self, using_key, access_token=access_token)
+    exception = check_and_attempt_upload(
+        self, using_key, access_token=access_token, print_progress=print_progress
+    )
     if exception is not None:
         self._delete_skip_storage()
         raise RuntimeError(exception)

lamindb/_collection.py

@@ -15,7 +15,12 @@ from anndata import AnnData
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core.hashing import hash_set
-from lnschema_core.models import Collection, CollectionArtifact, FeatureSet
+from lnschema_core.models import (
+    Collection,
+    CollectionArtifact,
+    FeatureManager,
+    FeatureSet,
+)
 from lnschema_core.types import VisibilityChoice
 
 from lamindb._artifact import update_attributes
@@ -45,6 +50,7 @@ def __init__(
     *args,
     **kwargs,
 ):
+    collection.features = FeatureManager(collection)
     if len(args) == len(collection._meta.concrete_fields):
         super(Collection, collection).__init__(*args, **kwargs)
         return None
@@ -103,9 +109,9 @@ def __init__(
             if meta._state.adding:
                 raise ValueError("Save meta artifact before creating collection!")
             if not feature_sets:
-                feature_sets = meta.features.feature_set_by_slot
+                feature_sets = meta.features._feature_set_by_slot
             else:
-                if len(meta.features.feature_set_by_slot) > 0:
+                if len(meta.features._feature_set_by_slot) > 0:
                     logger.info("overwriting feature sets linked to artifact")
     # we ignore collections in trash containing the same hash
     if hash is not None:
@@ -129,7 +135,7 @@ def __init__(
             existing_collection.transform = run.transform
         init_self_from_db(collection, existing_collection)
         update_attributes(collection, {"description": description, "name": name})
-        for slot, feature_set in collection.features.feature_set_by_slot.items():
+        for slot, feature_set in collection.features._feature_set_by_slot.items():
             if slot in feature_sets:
                 if not feature_sets[slot] == feature_set:
                     collection.feature_sets.remove(feature_set)
@@ -322,7 +328,7 @@ def delete(self, permanent: bool | None = None) -> None:
 
 
 # docstring handled through attach_func_to_class_method
-def save(self, transfer_labels: bool = False, using: str | None = None) -> None:
+def save(self, using: str | None = None) -> None:
     if self.artifact is not None:
         self.artifact.save()
     # we don't need to save feature sets again
@@ -331,21 +337,19 @@ def save(self, transfer_labels: bool = False, using: str | None = None) -> None:
     # we don't allow updating the collection of artifacts
     # if users want to update the set of artifacts, they
     # have to create a new collection
-    links = [
-        CollectionArtifact(collection_id=self.id, artifact_id=artifact.id)
-        for artifact in self._artifacts
-    ]
-    # the below seems to preserve the order of the list in the
-    # auto-incrementing integer primary
-    # merely using .unordered_artifacts.set(*...) doesn't achieve this
-    # we need ignore_conflicts=True so that this won't error if links already exist
-    CollectionArtifact.objects.bulk_create(links, ignore_conflicts=True)
+    if hasattr(self, "_artifacts"):
+        links = [
+            CollectionArtifact(collection_id=self.id, artifact_id=artifact.id)
+            for artifact in self._artifacts
+        ]
+        # the below seems to preserve the order of the list in the
+        # auto-incrementing integer primary
+        # merely using .unordered_artifacts.set(*...) doesn't achieve this
+        # we need ignore_conflicts=True so that this won't error if links already exist
+        CollectionArtifact.objects.bulk_create(links, ignore_conflicts=True)
     save_feature_set_links(self)
     if using is not None:
         logger.warning("using argument is ignored")
-    if transfer_labels:
-        for artifact in self._artifacts:
-            self.labels.add_from(artifact)
 
 
 # docstring handled through attach_func_to_class_method

lamindb/_feature_set.py

@@ -73,7 +73,7 @@ def __init__(self, *args, **kwargs):
     features_hash = hash_set({feature.uid for feature in features})
     feature_set = FeatureSet.filter(hash=features_hash).one_or_none()
     if feature_set is not None:
-        logger.success(f"loaded: {feature_set}")
+        logger.debug(f"loaded: {feature_set}")
         init_self_from_db(self, feature_set)
         return None
     else:

lamindb/_filter.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from lnschema_core import Artifact, Collection, Registry
+from lnschema_core import Artifact, Collection, Feature, Registry
 from lnschema_core.types import VisibilityChoice
 
 from lamindb import settings
@@ -14,7 +14,11 @@ def filter(Registry: type[Registry], **expressions) -> QuerySet:
         _using_key = expressions.pop("_using_key")
     if Registry in {Artifact, Collection}:
         # visibility is set to 0 unless expressions contains id or uid equality
-        if not ("id" in expressions or "uid" in expressions):
+        if not (
+            "id" in expressions
+            or "uid" in expressions
+            or "uid__startswith" in expressions
+        ):
             visibility = "visibility"
             if not any(e.startswith(visibility) for e in expressions):
                 expressions[

lamindb/_finish.py

@@ -63,6 +63,7 @@ def save_run_context_core(
     filepath: Path,
     transform_family: QuerySet | None = None,
     finished_at: bool = False,
+    from_cli: bool = False,
 ) -> str | None:
     import lamindb as ln
 
@@ -70,9 +71,10 @@ def save_run_context_core(
 
     # for scripts, things are easy
     is_consecutive = True
+    is_notebook = transform.type == TransformType.notebook
     source_code_path = filepath
     # for notebooks, we need more work
-    if transform.type == TransformType.notebook:
+    if is_notebook:
         try:
             import nbstripout
             from nbproject.dev import (
@@ -85,7 +87,9 @@ def save_run_context_core(
             )
             return None
         notebook_content = read_notebook(filepath)  # type: ignore
-        is_consecutive = check_consecutiveness(notebook_content)
+        is_consecutive = check_consecutiveness(
+            notebook_content, calling_statement="ln.finish()"
+        )
         if not is_consecutive:
             msg = "   Do you still want to proceed with finishing? (y/n) "
             if os.getenv("LAMIN_TESTING") is None:
@@ -106,13 +110,13 @@ def save_run_context_core(
         # in an existing storage location -> we want to move associated
         # artifacts into default storage and not register them in an existing
         # location
-        filepath_html_orig = filepath.with_suffix(".html")  # current location
-        filepath_html = ln_setup.settings.storage.cache_dir / filepath_html_orig.name
+        report_path_orig = filepath.with_suffix(".html")  # current location
+        report_path = ln_setup.settings.storage.cache_dir / report_path_orig.name
         # don't use Path.rename here because of cross-device link error
         # https://laminlabs.slack.com/archives/C04A0RMA0SC/p1710259102686969
         shutil.move(
-            filepath_html_orig,  # type: ignore
-            filepath_html,
+            report_path_orig,  # type: ignore
+            report_path,
         )
         # strip the output from the notebook to create the source code file
         # first, copy the notebook file to a temporary file in the cache
@@ -159,6 +163,8 @@ def save_run_context_core(
             else:
                 logger.warning("Please re-run `ln.track()` to make a new version")
                 return "rerun-the-notebook"
+        else:
+            logger.important("source code is already saved")
     else:
         source_code = ln.Artifact(
             source_code_path,
@@ -168,57 +174,73 @@ def save_run_context_core(
             visibility=0,  # hidden file
             run=False,
         )
-        source_code.save(upload=True)
+        source_code.save(upload=True, print_progress=False)
         transform.source_code = source_code
-        logger.success(f"saved transform.source_code: {transform.source_code}")
+        logger.debug(f"saved transform.source_code: {transform.source_code}")
 
     # track environment
-    filepath_env = ln_setup.settings.storage.cache_dir / f"run_env_pip_{run.uid}.txt"
-    if filepath_env.exists():
-        hash, _ = hash_file(filepath_env)
-        artifact = ln.Artifact.filter(hash=hash, visibility=0).one_or_none()
-        new_env_artifact = artifact is None
-        if new_env_artifact:
-            artifact = ln.Artifact(
-                filepath_env,
-                description="requirements.txt",
-                visibility=0,
-                run=False,
-            )
-            artifact.save(upload=True)
-        run.environment = artifact
-        if new_env_artifact:
-            logger.success(f"saved run.environment: {run.environment}")
+    env_path = ln_setup.settings.storage.cache_dir / f"run_env_pip_{run.uid}.txt"
+    if env_path.exists():
+        overwrite_env = True
+        if run.environment_id is not None and from_cli:
+            logger.important("run.environment is already saved")
+            overwrite_env = False
+        if overwrite_env:
+            hash, _ = hash_file(env_path)
+            artifact = ln.Artifact.filter(hash=hash, visibility=0).one_or_none()
+            new_env_artifact = artifact is None
+            if new_env_artifact:
+                artifact = ln.Artifact(
+                    env_path,
+                    description="requirements.txt",
+                    visibility=0,
+                    run=False,
+                )
+                artifact.save(upload=True, print_progress=False)
+            run.environment = artifact
+            if new_env_artifact:
+                logger.debug(f"saved run.environment: {run.environment}")
 
     # set finished_at
     if finished_at:
         run.finished_at = datetime.now(timezone.utc)
 
     # track report and set is_consecutive
-    if not transform.type == TransformType.notebook:
+    if not is_notebook:
         run.is_consecutive = True
         run.save()
     else:
         if run.report_id is not None:
-            logger.warning(
-                "there is already an existing report for this run, replacing it"
-            )
-            run.report.replace(filepath_html)
-            run.report.save(upload=True)
+            hash, _ = hash_file(report_path)  # ignore hash_type for now
+            if hash != run.report.hash:
+                if os.getenv("LAMIN_TESTING") is None:
+                    # in test, auto-confirm overwrite
+                    response = input(
+                        f"You are about to replace (overwrite) an existing run report (hash '{run.report.hash}'). Proceed? (y/n)"
+                    )
+                else:
+                    response = "y"
+                if response == "y":
+                    run.report.replace(report_path)
+                    run.report.save(upload=True)
+                else:
+                    logger.important("keeping old report")
+            else:
+                logger.important("report is already saved")
         else:
             report_file = ln.Artifact(
-                filepath_html,
+                report_path,
                 description=f"Report of run {run.uid}",
                 is_new_version_of=prev_report,
                 visibility=0,  # hidden file
                 run=False,
             )
-            report_file.save(upload=True)
+            report_file.save(upload=True, print_progress=False)
             run.report = report_file
         run.is_consecutive = is_consecutive
         run.save()
         transform.latest_report = run.report
-        logger.success(f"saved transform.latest_report: {transform.latest_report}")
+        logger.debug(f"saved transform.latest_report: {transform.latest_report}")
     transform.save()
 
     # finalize
@@ -227,6 +249,15 @@ def save_run_context_core(
         logger.important(
             f"go to: https://lamin.ai/{identifier}/transform/{transform.uid}"
         )
+        if not from_cli:
+            thing, name = (
+                ("notebook", "notebook.ipynb")
+                if is_notebook
+                else ("script", "script.py")
+            )
+            logger.important(
+                f"if you want to update your {thing} without re-running it, use `lamin save {name}`"
+            )
     # because run & transform changed, update the global run_context
     run_context.run = run
     run_context.transform = transform

lamindb/_from_values.py

@@ -18,12 +18,16 @@ def get_or_create_records(
     iterable: ListLike,
     field: StrField,
     *,
+    create: bool = False,
     from_public: bool = False,
     organism: Registry | str | None = None,
     public_source: Registry | None = None,
     mute: bool = False,
 ) -> list[Registry]:
     """Get or create records from iterables."""
+    Registry = field.field.model
+    if create:
+        return [Registry(**{field.field.name: value}) for value in iterable]
     upon_create_search_names = settings.upon_create_search_names
     feature: Feature = None
     organism = _get_organism_record(field, organism)
@@ -34,7 +38,6 @@ def get_or_create_records(
         kwargs["public_source"] = public_source
     settings.upon_create_search_names = False
     try:
-        Registry = field.field.model
         iterable_idx = index_iterable(iterable)
 
         # returns existing records & non-existing values
@@ -274,10 +277,13 @@ def index_iterable(iterable: Iterable) -> pd.Index:
     return idx[(idx != "") & (~idx.isnull())]
 
 
-def _print_values(names: Iterable, n: int = 20) -> str:
+def _print_values(names: Iterable, n: int = 20, quotes: bool = True) -> str:
     names = (name for name in names if name != "None")
     unique_names = list(dict.fromkeys(names))[:n]
-    print_values = ", ".join(f"'{name}'" for name in unique_names)
+    if quotes:
+        print_values = ", ".join(f"'{name}'" for name in unique_names)
+    else:
+        print_values = ", ".join(f"{name}" for name in unique_names)
     if len(unique_names) > n:
         print_values += ", ..."
     return print_values

lamindb/_parents.py

@@ -30,6 +30,8 @@ def _transform_emoji(transform: Transform):
 
 
 def _view(u):
+    from graphviz.backend import ExecutableNotFound
+
     try:
         if is_run_from_ipython:
             from IPython import get_ipython
@@ -39,10 +41,12 @@ def _view(u):
             if get_ipython().__class__.__name__ == "TerminalInteractiveShell":
                 return u.view()
             else:
-                display(u)
+                # call u._repr_mimebundle_() manually that exception gets raised properly and not just printed by
+                # call to display()
+                display(u._repr_mimebundle_(), raw=True)
         else:
             return u
-    except (FileNotFoundError, RuntimeError):  # pragma: no cover
+    except (FileNotFoundError, RuntimeError, ExecutableNotFound):  # pragma: no cover
         logger.error(
             "please install the graphviz executable on your system:\n  - Ubuntu: `sudo"
             " apt-get install graphviz`\n  - Windows:"
@@ -177,9 +181,11 @@ def _view_parents(
     )
     u.node(
         record.uid,
-        label=_record_label(record)
-        if record.__class__.__name__ == "Transform"
-        else _add_emoji(record, record_label),
+        label=(
+            _record_label(record)
+            if record.__class__.__name__ == "Transform"
+            else _add_emoji(record, record_label)
+        ),
         fillcolor=LAMIN_GREEN_LIGHTER,
     )
     if df_edges is not None:

lamindb/_query_manager.py

@@ -9,7 +9,7 @@ from lnschema_core.models import Registry
 
 from lamindb.core._settings import settings
 
-from .core._feature_manager import get_feature_set_by_slot
+from .core._feature_manager import get_feature_set_by_slot_
 
 if TYPE_CHECKING:
     from lnschema_core.types import StrField
@@ -107,7 +107,7 @@ class QueryManager(models.Manager):
                 source_field_name in {"artifact", "collection"}
                 and target_field_name == "feature_set"
             ):
-                return get_feature_set_by_slot(host=self.instance).get(item)
+                return get_feature_set_by_slot_(host=self.instance).get(item)
 
         except Exception:  # pragma: no cover
             return

lamindb/_query_set.py

@@ -99,9 +99,30 @@ class QuerySet(models.QuerySet, CanValidate):
     @doc_args(Registry.df.__doc__)
     def df(self, include: str | list[str] | None = None) -> pd.DataFrame:
         """{}."""
-        data = self.values()
-        keys = get_keys_from_df(data, self.model)
-        df = pd.DataFrame(self.values(), columns=keys)
+        # re-order the columns
+        exclude_field_names = ["created_at"]
+        field_names = [
+            field.name
+            for field in self.model._meta.fields
+            if (
+                not isinstance(field, models.ForeignKey)
+                and field.name not in exclude_field_names
+            )
+        ]
+        field_names += [
+            f"{field.name}_id"
+            for field in self.model._meta.fields
+            if isinstance(field, models.ForeignKey)
+        ]
+        for field_name in ["run_id", "created_at", "created_by_id", "updated_at"]:
+            if field_name in field_names:
+                field_names.remove(field_name)
+                field_names.append(field_name)
+        if field_names[0] != "uid" and "uid" in field_names:
+            field_names.remove("uid")
+            field_names.insert(0, "uid")
+        # create the dataframe
+        df = pd.DataFrame(self.values(), columns=field_names)
         # if len(df) > 0 and "updated_at" in df:
         #     df.updated_at = format_and_convert_to_local_time(df.updated_at)
         # if len(df) > 0 and "started_at" in df: