lamindb 0.74.0__py3-none-any.whl → 0.74.2__py3-none-any.whl

lamindb/core/_settings.py

@@ -9,11 +9,13 @@ from lamindb_setup._set_managed_storage import set_managed_storage
 from lamindb_setup.core._settings import settings as setup_settings
 from lamindb_setup.core._settings_instance import sanitize_git_repo_url
 
-from ._transform_settings import TransformSettings, transform
+from .subsettings._creation_settings import CreationSettings, creation_settings
+from .subsettings._transform_settings import TransformSettings, transform_settings
 
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from lamindb_setup.core._settings_storage import StorageSettings
     from upath import UPath
 
 VERBOSITY_TO_INT = {
@@ -40,29 +42,15 @@ class Settings:
         logger.set_verbosity(self._verbosity_int)
         self._sync_git_repo: str | None = git_repo
 
-    upon_artifact_create_if_hash_exists: Literal[
-        "warn_return_existing", "error", "warn_create_new"
-    ] = "warn_return_existing"
-    """Behavior if file hash exists (default `"warn_return_existing"`).
-
-    One of `["warn_return_existing", "error", "warn_create_new"]`.
-
-    FAQ: :doc:`/faq/idempotency`
-    """
-    upon_file_create_skip_size_hash: bool = False
-    """To speed up registering high numbers of files (default `False`).
-
-    This bypasses queries for size and hash to AWS & GCP.
-
-    It speeds up file creation by about a factor 100.
-    """
-    upon_create_search_names: bool = True
-    """To speed up creating Registry objects (default `True`).
+    @property
+    def creation(self) -> CreationSettings:
+        """Record creation settings.
 
-    If `True`, search for alternative names.
+        For example, `ln.settings.creation.search_names = False` will disable
+        searching for records with similar names during creation.
+        """
+        return creation_settings
 
-    FAQ: :doc:`/faq/idempotency`
-    """
     track_run_inputs: bool = True
     """Track files as input upon `.load()`, `.cache()` and `.backed()`.
 
@@ -70,14 +58,6 @@ class Settings:
 
     FAQ: :doc:`/faq/track-run-inputs`
     """
-    silence_file_run_transform_warning: bool = False
-    """Silence warning about missing run & transform during file creation."""
-    artifact_use_virtual_keys: bool = True
-    """Treat `key` parameter in :class:`~lamindb.Artifact` as virtual.
-
-    If `True`, the `key` is **not** used to construct file paths, but file paths are
-    based on the `uid` of artifact.
-    """
     __using_key: str | None = None
     _using_storage: str | None = None
 
@@ -101,8 +81,17 @@ class Settings:
 
     @property
     def transform(self) -> TransformSettings:
-        """Transform settings."""
-        return transform
+        """Transform settings.
+
+        For example::
+
+            ln.settings.transform.stem_uid = "FPnfDtJz8qbE"  # defines version family
+            ln.settings.transform.version = "1"              # defines version
+            ln.settings.transform.name = "My good script"    # semantic name
+
+        The first two are typically auto-generated by :func:`~lamindb.track`.
+        """
+        return transform_settings
 
     @property
     def sync_git_repo(self) -> str | None:
@@ -116,18 +105,26 @@ class Settings:
     def sync_git_repo(self, value) -> None:
         """Sync transforms with scripts in git repository.
 
-        Provide the full git repo URL.
+        For example: `ln.sync_git_repo = https://github.com/laminlabs/redun-lamin`
         """
         self._sync_git_repo = sanitize_git_repo_url(value)
-        assert self._sync_git_repo.startswith("https://")
+        if not self._sync_git_repo.startswith("https://"):  # pragma: nocover
+            raise ValueError("git repository URL must start with 'https://'.")
 
     @property
-    def storage(self) -> Path | UPath:
-        """Default storage location (a path to its root).
+    def storage(self) -> StorageSettings:
+        """Default storage location.
 
         Examples:
 
-        You can switch to another managed storage location via:
+        >>> ln.settings.storage
+        StorageSettings(root='s3://my-bucket', uid='j7MaPxtLxPeE')
+
+        >>> ln.settings.storage.root
+        UPath('s3://my-bucket')
+
+        You can switch the default storage location to another managed storage
+        location by passing a string:
 
         >>> ln.settings.storage = "s3://some-bucket"
 
@@ -139,7 +136,7 @@ class Settings:
         >>> )
         >>> ln.settings.storage = "s3://some-bucket", kwargs
         """
-        return self._storage_settings.root
+        return self._storage_settings
 
     @storage.setter
     def storage(self, path_kwargs: str | Path | UPath | tuple[str | UPath, Mapping]):
@@ -150,16 +147,14 @@ class Settings:
         set_managed_storage(path, **kwargs)
 
     @property
-    def storage_local(self) -> Path:
+    def storage_local(self) -> StorageSettings:
         """An additional local default storage (a path to its root).
 
         Is only available if :attr:`~lamindb.setup.core.InstanceSettings.keep_artifacts_local` is enabled.
 
         Guide: :doc:`faq/keep-artifacts-local`
-
-        Shortcut for: `ln.setup.settings.instance.storage_local.root`
         """
-        return ln_setup.settings.instance.storage_local.root
+        return ln_setup.settings.instance.storage_local
 
     @storage_local.setter
     def storage_local(self, local_root: Path):
@@ -167,14 +162,14 @@ class Settings:
 
     @property
     def verbosity(self) -> str:
-        """Logger verbosity (default 'warning').
-
-        - 'error': ❌ only show error messages
-        - 'warning': ❗ also show warning messages
-        - 'success': ✅ also show success and save messages
-        - 'info': 💡 also show info messages
-        - 'hint': 💡 also show hint messages
-        - 'debug': 🐛 also show detailed debug messages
+        """Logger verbosity (default `'warning'`).
+
+        - `'error'`: ❌ only show error messages
+        - `'warning'`: ❗ also show warning messages
+        - `'success'`: ✅ also show success and save messages
+        - `'info'`: 💡 also show info messages
+        - `'hint'`: 💡 also show hint messages
+        - `'debug'`: 🐛 also show detailed debug messages
         """
         return VERBOSITY_TO_STR[self._verbosity_int]
 

lamindb/core/_sync_git.py

@@ -24,8 +24,7 @@ def get_git_repo_from_remote() -> Path:
         f"running outside of synched git repo, cloning {repo_url} into {repo_dir}"
     )
     result = subprocess.run(
-        f"git clone --depth 10 {repo_url}.git",
-        shell=True,
+        ["git", "clone", "--depth", "10", f"{repo_url}.git"],
         capture_output=True,
         cwd=setup_settings.storage.cache_dir,
     )
@@ -36,8 +35,7 @@ def get_git_repo_from_remote() -> Path:
 
 def check_local_git_repo() -> bool:
     result = subprocess.run(
-        "git config --get remote.origin.url",
-        shell=True,
+        ["git", "config", "--get remote.origin.url"],
         capture_output=True,
     )
     result_str = result.stdout.decode().strip()
@@ -55,10 +53,9 @@ def check_local_git_repo() -> bool:
 
 
 def get_git_commit_hash(blob_hash: str, repo_dir: Path | None = None) -> str | None:
-    command = f"git log --find-object={blob_hash} --pretty=format:%H"
+    command = ["git", "log", f"--find-object={blob_hash}", "--pretty=format:%H"]
     result = subprocess.run(
         command,
-        shell=True,
         capture_output=True,
         cwd=repo_dir,
     )
@@ -68,7 +65,7 @@ def get_git_commit_hash(blob_hash: str, repo_dir: Path | None = None) -> str | N
     if commit_hash == "" or result.returncode == 1:
         return None
     else:
-        assert (
+        assert (  # noqa: S101
             len(commit_hash) == 40
         ), f"commit hash |{commit_hash}| is not 40 characters long"
         return commit_hash
@@ -82,21 +79,34 @@ def get_filepath_within_git_repo(
     # from anywhere in the repo, hence, let's get the root
     repo_root = (
         subprocess.run(
-            "git rev-parse --show-toplevel",
-            shell=True,
+            ["git", "rev-parse", "--show-toplevel"],
             capture_output=True,
             cwd=repo_dir,
         )
         .stdout.decode()
         .strip()
     )
-    command = f"git ls-tree -r {commit_hash} | grep -E {blob_hash}"
+    # Run the git commands separately to circumvent spawning a shell
+    git_command = ["git", "ls-tree", "-r", commit_hash]
+    git_process = subprocess.Popen(
+        git_command,
+        stdout=subprocess.PIPE,
+        cwd=repo_root,
+    )
+
+    grep_command = ["grep", "-E", blob_hash]
     result = subprocess.run(
-        command,
-        shell=True,
+        grep_command,
+        stdin=git_process.stdout,
         capture_output=True,
         cwd=repo_root,
     )
+
+    # Close the stdout to allow git_process to receive a SIGPIPE if grep_command exits
+    git_process.stdout.close()
+    git_process.wait()
+
+    command = " ".join(git_command) + " | " + " ".join(grep_command)
     if result.returncode != 0 and result.stderr.decode() != "":
         raise RuntimeError(f"{command}\n{result.stderr.decode()}")
     if len(result.stdout.decode()) == 0:

lamindb/core/_track_environment.py

@@ -15,7 +15,11 @@ def track_environment(run: Run) -> None:
     # create a requirements.txt
     # we don't create a conda environment.yml mostly for its slowness
     try:
-        result = subprocess.run(f"pip freeze > {str(filepath)}", shell=True)
+        with open(filepath, "w") as f:
+            result = subprocess.run(
+                ["pip", "freeze"],
+                stdout=f,
+            )
     except OSError as e:
         result = None
         logger.warning(f"could not run pip freeze with error {e}")

lamindb/core/datasets/_core.py

@@ -91,7 +91,7 @@ def file_tsv_rnaseq_nfcore_salmon_merged_gene_counts(
 
 def file_fastq(in_storage_root=False) -> Path:
     """Mini mock fastq artifact."""
-    basedir = Path() if not in_storage_root else settings.storage
+    basedir = Path() if not in_storage_root else settings.storage.root
     filepath = basedir / "input.fastq.gz"
     with open(filepath, "w") as f:
         f.write("Mock fastq artifact.")
@@ -100,7 +100,7 @@ def file_fastq(in_storage_root=False) -> Path:
 
 def file_bam(in_storage_root=False) -> Path:  # pragma: no cover
     """Mini mock bam artifact."""
-    basedir = Path() if not in_storage_root else settings.storage
+    basedir = Path() if not in_storage_root else settings.storage.root
     filepath = basedir / "output.bam"
     with open(filepath, "w") as f:
         f.write("Mock bam artifact.")
@@ -109,7 +109,7 @@ def file_bam(in_storage_root=False) -> Path:  # pragma: no cover
 
 def file_mini_csv(in_storage_root=False) -> Path:
     """Mini csv artifact."""
-    basedir = Path() if not in_storage_root else settings.storage
+    basedir = Path() if not in_storage_root else settings.storage.root
     filepath = basedir / "mini.csv"
     df = pd.DataFrame([1, 2, 3], columns=["test"])
     df.to_csv(filepath, index=False)

lamindb/core/fields.py

@@ -1,6 +1,6 @@
 """Fields.
 
-The field accessor of a :class:`~lamindb.core.Registry`:
+The field accessor of a :class:`~lamindb.core.Record`:
 
 .. autosummary::
    :toctree: .

lamindb/core/schema.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
 from django.db.models import ManyToManyField
-from lnschema_core.models import Feature, FeatureSet, LinkORM, Registry
+from lnschema_core.models import Feature, FeatureSet, LinkORM, Record
 
 
-def dict_schema_name_to_model_name(orm: type[Registry]) -> dict[str, Registry]:
+def dict_schema_name_to_model_name(orm: type[Record]) -> dict[str, Record]:
     d: dict = {
         i.related_model.__get_name_with_schema__(): i.related_model
         for i in orm._meta.related_objects
@@ -21,9 +21,9 @@ def dict_schema_name_to_model_name(orm: type[Registry]) -> dict[str, Registry]:
 
 
 def dict_related_model_to_related_name(
-    orm: type[Registry], links: bool = False
+    orm: type[Record], links: bool = False
 ) -> dict[str, str]:
-    def include(model: Registry):
+    def include(model: Record):
         return not links != issubclass(model, LinkORM)
 
     related_objects = orm._meta.related_objects + orm._meta.many_to_many
@@ -39,7 +39,7 @@ def dict_related_model_to_related_name(
     return d
 
 
-def get_related_name(features_type: type[Registry]) -> str:
+def get_related_name(features_type: type[Record]) -> str:
     candidates = [
         field.related_name
         for field in FeatureSet._meta.related_objects
@@ -49,7 +49,7 @@ def get_related_name(features_type: type[Registry]) -> str:
         raise ValueError(
             f"Can't create feature sets from {features_type.__name__} because it's not"
             " related to it!\nYou need to create a link model between FeatureSet and"
-            " your Registry in your custom schema.\nTo do so, add a"
+            " your Record in your custom schema.\nTo do so, add a"
             " line:\nfeature_sets = models.ManyToMany(FeatureSet,"
             " related_name='mythings')\n"
         )

lamindb/core/storage/_backed_access.py

@@ -28,6 +28,8 @@ if TYPE_CHECKING:
     from pathlib import Path
 
     from fsspec.core import OpenFile
+    from tiledbsoma import Collection as SOMACollection
+    from tiledbsoma import Experiment as SOMAExperiment
 
 anndata_version_parse = version.parse(anndata_version)
 
@@ -100,7 +102,7 @@ def _records_to_df(obj):
         return obj
 
 
-class AccessRegistry:
+class AccessRecord:
     def __init__(self):
         self._registry = {}
         self._openers = {}
@@ -141,7 +143,7 @@ class AccessRegistry:
 
 
 # storage specific functions should be registered and called through the registry
-registry = AccessRegistry()
+registry = AccessRecord()
 
 
 @registry.register_open("h5py")
@@ -207,8 +209,10 @@ def safer_read_partial(elem, indices):
             try:
                 ds = CSRDataset(elem)
                 result = _subset_sparse(ds, indices)
-            except Exception:
-                pass
+            except Exception as e:
+                logger.debug(
+                    f"Encountered an exception while attempting to subset a sparse dataset by indices.\n{e}"
+                )
         if result is None:
             raise ValueError(
                 "Can not get a subset of the element of type"
@@ -305,8 +309,10 @@ if ZARR_INSTALLED:
                 try:
                     ds = CSRDataset(elem)
                     return _subset_sparse(ds, indices)
-                except Exception:
-                    pass
+                except Exception as e:
+                    logger.debug(
+                        f"Encountered an exception while attempting to subset a sparse dataset by indices.\n{e}"
+                    )
             raise ValueError(
                 "Can not get a subset of the element of type"
                 f" {type(elem).__name__} with an empty spec."
@@ -734,24 +740,62 @@ class BackedAccessor:
 
 def backed_access(
     artifact_or_filepath: Artifact | Path, using_key: str | None = None
-) -> AnnDataAccessor | BackedAccessor:
+) -> AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment:
     if isinstance(artifact_or_filepath, Artifact):
         filepath = filepath_from_artifact(artifact_or_filepath, using_key=using_key)
     else:
         filepath = artifact_or_filepath
     name = filepath.name
+    suffix = filepath.suffix
+
+    if name == "soma" or suffix == ".tiledbsoma":
+        try:
+            import tiledbsoma as soma
+        except ImportError as e:
+            raise ImportError(
+                "Please install tiledbsoma: pip install tiledbsoma"
+            ) from e
+        filepath_str = filepath.as_posix()
+        if filepath.protocol == "s3":
+            from lamindb_setup.core._settings_storage import get_storage_region
+
+            region = get_storage_region(filepath_str)
+            tiledb_config = {"vfs.s3.region": region}
+            storage_options = filepath.storage_options
+            if "key" in storage_options:
+                tiledb_config["vfs.s3.aws_access_key_id"] = storage_options["key"]
+            if "secret" in storage_options:
+                tiledb_config["vfs.s3.aws_secret_access_key"] = storage_options[
+                    "secret"
+                ]
+            if "token" in storage_options:
+                tiledb_config["vfs.s3.aws_session_token"] = storage_options["token"]
+            ctx = soma.SOMATileDBContext(tiledb_config=tiledb_config)
+            # this is a strange bug
+            # for some reason iterdir futher gives incorrect results
+            # if cache is not invalidated
+            # instead of obs and ms it gives ms and ms in the list of names
+            filepath.fs.invalidate_cache()
+        else:
+            ctx = None
 
-    if filepath.suffix in (".h5", ".hdf5", ".h5ad"):
+        soma_objects = [obj.name for obj in filepath.iterdir()]
+        if "obs" in soma_objects and "ms" in soma_objects:
+            SOMAType = soma.Experiment
+        else:
+            SOMAType = soma.Collection
+        return SOMAType.open(filepath_str, context=ctx)
+    elif suffix in {".h5", ".hdf5", ".h5ad"}:
         conn, storage = registry.open("h5py", filepath)
-    elif filepath.suffix == ".zarr":
+    elif suffix == ".zarr":
         conn, storage = registry.open("zarr", filepath)
     else:
         raise ValueError(
-            "object should have .h5, .hdf5, .h5ad, .zarr suffix, not"
-            f" {filepath.suffix}."
+            "object should have .h5, .hdf5, .h5ad, .zarr, .tiledbsoma suffix, not"
+            f" {suffix}."
         )
 
-    if filepath.suffix == ".h5ad":
+    if suffix == ".h5ad":
         return AnnDataAccessor(conn, storage, name)
     else:
         if get_spec(storage).encoding_type == "anndata":

lamindb/core/storage/paths.py

@@ -47,7 +47,7 @@ def auto_storage_key_from_artifact(artifact: Artifact):
 
 
 def auto_storage_key_from_artifact_uid(uid: str, suffix: str, is_dir: bool) -> str:
-    assert isinstance(suffix, str)
+    assert isinstance(suffix, str)  # noqa: S101 Suffix cannot be None.
     if is_dir:
         uid_storage = uid[:16]  # 16 chars, leave 4 chars for versioning
     else:
@@ -72,7 +72,7 @@ def attempt_accessing_path(
             storage_settings = settings._storage_settings
         else:
             storage_settings = StorageSettings(
-                settings.storage, access_token=access_token
+                settings.storage.root, access_token=access_token
             )
     else:
         if artifact._state.db not in ("default", None) and using_key is None:
@@ -143,8 +143,8 @@ def delete_storage(
     """Delete arbitrary artifact."""
     # TODO is_relative_to is not available in 3.8 and deprecated since 3.12
     # replace with check_path_is_child_of_root but this needs to first be debugged
-    # if not check_path_is_child_of_root(storagepath, settings.storage):
-    if not storagepath.is_relative_to(settings.storage):  # type: ignore
+    # if not check_path_is_child_of_root(storagepath, settings.storage.root):
+    if not storagepath.is_relative_to(settings.storage.root):  # type: ignore
         allow_delete = False
         if setup_settings.instance.keep_artifacts_local:
             allow_delete = storagepath.is_relative_to(  # type: ignore

lamindb/core/subsettings/__init__.py

@@ -0,0 +1,12 @@
+"""Sub settings.
+
+.. autosummary::
+   :toctree: .
+
+   TransformSettings
+   CreationSettings
+
+"""
+
+from ._creation_settings import CreationSettings
+from ._transform_settings import TransformSettings

lamindb/core/subsettings/_creation_settings.py

@@ -0,0 +1,38 @@
+from typing import Literal
+
+
+class CreationSettings:
+    artifact_if_hash_exists: Literal[
+        "warn_return_existing", "error", "warn_create_new"
+    ] = "warn_return_existing"
+    """Behavior if file hash exists (default `"warn_return_existing"`).
+
+    One of `["warn_return_existing", "error", "warn_create_new"]`.
+
+    FAQ: :doc:`/faq/idempotency`
+    """
+    artifact_skip_size_hash: bool = False
+    """To speed up registering high numbers of files (default `False`).
+
+    This bypasses queries for size and hash to AWS & GCP.
+
+    It speeds up file creation by about a factor 100.
+    """
+    search_names: bool = True
+    """To speed up creating records (default `True`).
+
+    If `True`, search for alternative names.
+
+    FAQ: :doc:`/faq/idempotency`
+    """
+    artifact_silence_missing_run_warning: bool = False
+    """Silence warning about missing run & transform during artifact creation."""
+    _artifact_use_virtual_keys: bool = True
+    """Treat `key` parameter in :class:`~lamindb.Artifact` as virtual.
+
+    If `True`, the `key` is **not** used to construct file paths, but file paths are
+    based on the `uid` of artifact.
+    """
+
+
+creation_settings = CreationSettings()

lamindb/core/subsettings/_transform_settings.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+
+class TransformSettings:
+    stem_uid: str | None = None
+    """Defines the version family of the transform.
+
+    For example, all notebooks of the same family have a uid that starts with
+    `"FPnfDtJz8qbE"`.
+
+    The full uids of the notebooks in this family are of form
+    `"{stem_uid}{suffix_uid}"` where the `suffix_uid` encodes the semantic
+    `version`.
+    """
+    version: str | None = None
+    """The version."""
+    name: str | None = None
+    """A name like a notebook or script title."""
+
+
+transform_settings = TransformSettings()

lamindb/core/versioning.py

@@ -97,7 +97,7 @@ def get_uid_from_old_version(
     version: str | None = None,
     using_key: str | None = None,
 ) -> tuple[str, str]:
-    """{}."""
+    """{}"""  # noqa: D415
     msg = ""
     if is_new_version_of.version is None:
         previous_version = "1"

lamindb/integrations/_vitessce.py

@@ -39,9 +39,10 @@ def save_vitessce_config(vitessce_config: VitessceConfig, description: str) -> A
             if "url" not in file:
                 raise ValueError("Each file must have a 'url' key.")
             filename = file["url"].split("/")[-1]
-            assert filename.endswith(
-                (".anndata.zarr", ".spatialdata.zarr", ".ome.zarr")
-            )
+            if not filename.endswith((".anndata.zarr", ".zarr", ".ome.zarr")):
+                logger.warning(
+                    "filename should end with '.anndata.zarr', '.zarr', or '.ome.zarr'."
+                )
             filestem = (
                 filename.replace(".anndata.zarr", "")
                 .replace(".spatialdata.zarr", "")

{lamindb-0.74.0.dist-info → lamindb-0.74.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.74.0
+Version: 0.74.2
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,24 +9,22 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.70.0
-Requires-Dist: lamindb_setup==0.73.2
+Requires-Dist: lnschema_core==0.70.5
+Requires-Dist: lamindb_setup==0.74.1
 Requires-Dist: lamin_utils==0.13.2
-Requires-Dist: lamin_cli==0.14.0
+Requires-Dist: lamin_cli==0.15.0
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
-Requires-Dist: anndata>=0.8.0,<0.10.8
+Requires-Dist: anndata>=0.8.0,<=0.10.8
 Requires-Dist: scipy<1.13.0rc1
 Requires-Dist: fsspec
 Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
-Requires-Dist: psutil
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.44.0 ; extra == "bionty"
-Requires-Dist: pandas<2 ; extra == "dev"
+Requires-Dist: bionty==0.44.2 ; extra == "bionty"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
 Requires-Dist: laminci>=0.3 ; extra == "dev"
@@ -59,7 +57,7 @@ Provides-Extra: zarr
 
 # LaminDB - A data framework for biology
 
-- Manage storage & databases with a unified Python API ("lakehouse").
+- Manage data & metadata with a unified Python API ("lakehouse").
 - Track data lineage across notebooks & pipelines.
 - Integrate registries for experimental metadata & in-house ontologies.
 - Validate, standardize & annotate.