linkml-store 0.1.6__py3-none-any.whl → 0.1.8__py3-none-any.whl

linkml_store/api/database.py

@@ -1,11 +1,14 @@
 import logging
 from abc import ABC
+from collections import defaultdict
 from copy import copy
 from pathlib import Path
-from typing import TYPE_CHECKING, ClassVar, Dict, Iterator, Optional, Sequence, Type, Union
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterator, Optional, Sequence, Type, Union
+
+from linkml_store.utils.format_utils import load_objects, render_output
 
 try:
-    from linkml.validator.report import ValidationResult
+    from linkml.validator.report import Severity, ValidationResult
 except ImportError:
     ValidationResult = None
 
@@ -26,13 +29,33 @@ class Database(ABC):
     """
     A Database provides access to named collections of data.
 
-    Examples
-    --------
+    A database object is owned by a :ref:`Client`. The database
+    object uses a :ref:`handle` to know what kind of external
+    dataase system to connect to (e.g. duckdb, mongodb). The handle
+    is a string ``<DatabaseType>:<LocalLocator>``
+
+    The
+    database object may also have an :ref:`alias` that is mapped
+    to the handle.
+
+    Attaching a database
+    --------------------
     >>> from linkml_store.api.client import Client
     >>> client = Client()
-    >>> db = client.attach_database("duckdb", alias="test")
+    >>> db = client.attach_database("duckdb:///:memory:", alias="test")
+
+    We can check the value of the handle:
+
     >>> db.handle
     'duckdb:///:memory:'
+
+    The alias can be used to retrieve the database object from the client
+
+    >>> assert db == client.get_database("test")
+
+    Creating a collection
+    ---------------------
+
     >>> collection = db.create_collection("Person")
     >>> len(db.list_collections())
     1
@@ -56,6 +79,11 @@ class Database(ABC):
     """
 
     _schema_view: Optional[SchemaView] = None
+    """Schema for the database. May be transformed."""
+
+    _original_schema_view: Optional[SchemaView] = None
+    """If a schema must be transformed, then the original is stored here."""
+
     _collections: Optional[Dict[str, Collection]] = None
     parent: Optional["Client"] = None
     metadata: Optional[DatabaseConfig] = None
@@ -100,6 +128,8 @@ class Database(ABC):
         return self
 
     def _initialize_collections(self):
+        if not self.metadata.collections:
+            return
         for name, collection_config in self.metadata.collections.items():
             alias = collection_config.alias
             typ = collection_config.type
@@ -126,15 +156,46 @@ class Database(ABC):
 
     @property
     def recreate_if_exists(self) -> bool:
+        """
+        Return whether to recreate the database if it already exists.
+
+        :return:
+        """
         return self.metadata.recreate_if_exists
 
     @property
     def handle(self) -> str:
+        """
+        Return the database handle.
+
+        Examples:
+
+        - ``duckdb:///:memory:``
+        - ``duckdb:///tmp/test.db``
+        - ``mongodb://localhost:27017/``
+
+        :return:
+        """
         return self.metadata.handle
 
-    def store(self, obj: Dict[str, str], **kwargs):
+    @property
+    def alias(self):
+        return self.metadata.alias
+
+    def store(self, obj: Dict[str, Any], **kwargs):
         """
-        Store an object in the database
+        Store an object in the database.
+
+        The object is assumed to be a Dictionary of Collections.
+
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> db.store({"persons": [{"id": "P1", "name": "John", "age_in_years": 30}]})
+        >>> collection = db.get_collection("persons")
+        >>> qr = collection.find()
+        >>> qr.num_rows
+        1
 
         :param obj: object to store
         :param kwargs: additional arguments
@@ -143,6 +204,7 @@ class Database(ABC):
         roots = [c for c in sv.all_classes().values() if c.tree_root]
         root = roots[0] if roots else None
         for k, v in obj.items():
+            logger.info(f"Storing collection {k}")
             if root:
                 slot = sv.induced_slot(k, root.name)
                 if not slot:
@@ -157,20 +219,28 @@ class Database(ABC):
             if not v:
                 continue
             if slot:
-                collection = self.get_collection(slot.range, create_if_not_exists=True)
+                logger.debug(f"Aligning to existing slot: {slot.name} range={slot.range}")
+                collection = self.get_collection(slot.name, type=slot.range, create_if_not_exists=True)
             else:
                 collection = self.get_collection(k, create_if_not_exists=True)
+            logger.debug(f"Replacing using {collection.alias} {collection.target_class_name}")
             collection.replace(v)
 
     def commit(self, **kwargs):
         """
-        Commit any pending changes to the database
+        Commit pending changes to the database.
+
+        :param kwargs:
+        :return:
         """
         raise NotImplementedError()
 
     def close(self, **kwargs):
         """
-        Close the database and all connection objects
+        Close the database.
+
+        :param kwargs:
+        :return:
         """
         raise NotImplementedError()
 
@@ -187,15 +257,27 @@ class Database(ABC):
         **kwargs,
     ) -> Collection:
         """
-        Create a new collection
+        Create a new collection in the current database.
+
+        The collection must have a *Type*, and may have an *Alias*.
+
+        Examples:
 
         >>> from linkml_store.api.client import Client
         >>> client = Client()
         >>> db = client.attach_database("duckdb", alias="test")
-        >>> collection = db.create_collection("Person")
-        >>> collection.name
+        >>> collection = db.create_collection("Person", alias="persons")
+        >>> collection.alias
+        'persons'
+        >>> collection.target_class_name
         'Person'
 
+        If alias is not provided, it defaults to the name of the type.
+
+        >>> collection = db.create_collection("Organization")
+        >>> collection.alias
+        'Organization'
+
         :param name: name of the collection
         :param alias: alias for the collection
         :param metadata: metadata for the collection
@@ -204,9 +286,10 @@ class Database(ABC):
         """
         if not name:
             raise ValueError(f"Collection name must be provided: alias: {alias} metadata: {metadata}")
-        # collection_cls = self._collection_class
         collection_cls = self.collection_class
         collection = collection_cls(name=name, alias=alias, parent=self, metadata=metadata)
+        if metadata and metadata.source_location:
+            collection.load_from_source()
         if metadata and metadata.attributes:
             sv = self.schema_view
             schema = sv.schema
@@ -265,7 +348,9 @@ class Database(ABC):
         """
         return [c.name for c in self.list_collections(**kwargs)]
 
-    def get_collection(self, name: str, create_if_not_exists=True, **kwargs) -> "Collection":
+    def get_collection(
+        self, name: str, type: Optional[str] = None, create_if_not_exists=True, **kwargs
+    ) -> "Collection":
         """
         Get a named collection.
 
@@ -283,14 +368,19 @@ class Database(ABC):
         KeyError: 'Collection NonExistent does not exist'
 
         :param name: name of the collection
+        :param type: target class name
         :param create_if_not_exists: create the collection if it does not exist
 
         """
         if not self._collections:
+            logger.debug("Initializing collections")
             self.init_collections()
         if name not in self._collections.keys():
             if create_if_not_exists:
-                self._collections[name] = self.create_collection(name)
+                if type is None:
+                    type = name
+                logger.debug(f"Creating new collection: {name} kwargs: {kwargs}")
+                self._collections[name] = self.create_collection(type, alias=name, **kwargs)
             else:
                 raise KeyError(f"Collection {name} does not exist")
         return self._collections[name]
@@ -333,7 +423,29 @@ class Database(ABC):
     @property
     def schema_view(self) -> SchemaView:
         """
-        Return a schema view for the named collection
+        Return a schema view for the named collection.
+
+        If no explicit schema is provided, this will generalize one
+
+        Induced schema example:
+
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person", alias="persons")
+        >>> collection.insert([{"id": "P1", "name": "John", "age_in_years": 25}])
+        >>> schema_view = db.schema_view
+        >>> cd = schema_view.get_class("Person")
+        >>> cd.attributes["id"].range
+        'string'
+        >>> cd.attributes["age_in_years"].range
+        'integer'
+
+        We can reuse the same class:
+
+        >>> collection2 = db.create_collection("Person", alias="other_persons")
+        >>> collection2.class_definition().attributes["age_in_years"].range
+        'integer'
         """
         if not self._schema_view:
             self._initialize_schema()
@@ -341,14 +453,61 @@ class Database(ABC):
             self._schema_view = self.induce_schema_view()
         return self._schema_view
 
-    def set_schema_view(self, schema_view: SchemaView):
+    def set_schema_view(self, schema_view: Union[str, Path, SchemaView]):
         """
         Set the schema view for the database.
 
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> sv = SchemaView("tests/input/countries/countries.linkml.yaml")
+        >>> db.set_schema_view(sv)
+        >>> cd = db.schema_view.schema.classes["Country"]
+        >>> sorted(cd.slots)
+        ['capital', 'code', 'continent', 'languages', 'name']
+        >>> induced_slots = {s.name: s for s in sv.class_induced_slots("Country")}
+        >>> sorted(induced_slots.keys())
+        ['capital', 'code', 'continent', 'languages', 'name']
+        >>> induced_slots["code"].identifier
+        True
+
+        Creating a new collection will align with the schema view:
+
+        >>> collection = db.create_collection("Country", "all_countries")
+        >>> sorted(collection.class_definition().slots)
+        ['capital', 'code', 'continent', 'languages', 'name']
+
         :param schema_view:
         :return:
         """
+        if isinstance(schema_view, Path):
+            schema_view = str(schema_view)
+        if isinstance(schema_view, str):
+            schema_view = SchemaView(schema_view)
         self._schema_view = schema_view
+        if not self._collections:
+            return
+        # align with induced schema
+        roots = [c for c in schema_view.all_classes().values() if c.tree_root]
+        if len(roots) == 0:
+            all_ranges = set()
+            for cn in schema_view.all_classes():
+                for slot in schema_view.class_induced_slots(cn):
+                    if slot.range:
+                        all_ranges.add(slot.range)
+            roots = [
+                c
+                for c in schema_view.all_classes().values()
+                if not all_ranges.intersection(schema_view.class_ancestors(c.name, reflexive=True))
+            ]
+        if len(roots) == 1:
+            root = roots[0]
+            for slot in schema_view.class_induced_slots(root.name):
+                inlined = slot.inlined or slot.inlined_as_list
+                if inlined and slot.range:
+                    if slot.name in self._collections:
+                        coll = self._collections[slot.name]
+                        coll.metadata.type = slot.range
 
     def load_schema_view(self, path: Union[str, Path]):
         """
@@ -358,6 +517,21 @@ class Database(ABC):
         >>> client = Client()
         >>> db = client.attach_database("duckdb", alias="test")
         >>> db.load_schema_view("tests/input/countries/countries.linkml.yaml")
+        >>> sv = db.schema_view
+        >>> cd = sv.schema.classes["Country"]
+        >>> sorted(cd.slots)
+        ['capital', 'code', 'continent', 'languages', 'name']
+        >>> induced_slots = {s.name: s for s in sv.class_induced_slots("Country")}
+        >>> sorted(induced_slots.keys())
+        ['capital', 'code', 'continent', 'languages', 'name']
+        >>> induced_slots["code"].identifier
+        True
+
+        Creating a new collection will align with the schema view:
+
+        >>> collection = db.create_collection("Country", "all_countries")
+        >>> sorted(collection.class_definition().slots)
+        ['capital', 'code', 'continent', 'languages', 'name']
 
         :param path:
         :return:
@@ -392,14 +566,126 @@ class Database(ABC):
         """
         Validate the contents of the database.
 
+        An an example, let's create a database with a predefined schema
+        from the countries.linkml.yaml file:
+
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> db.load_schema_view("tests/input/countries/countries.linkml.yaml")
+
+        Let's introspect the schema to see what slots are applicable for the class "Country":
+
+        >>> sv = db.schema_view
+        >>> for slot in sv.class_induced_slots("Country"):
+        ...     print(slot.name, slot.range, slot.required)
+        name string True
+        code string True
+        capital string True
+        continent string True
+        languages Language None
+
+        Next we'll create a collection, binding it to the target class "Country", and insert
+        valid data:
+
+        >>> collection = db.create_collection("Country", "all_countries")
+        >>> obj = {"code": "US", "name": "United States", "continent": "North America", "capital": "Washington, D.C."}
+        >>> collection.insert([obj])
+        >>> list(db.iter_validate_database())
+        []
+
+        Now let's insert some invalid data (missing required fields)
+
+        >>> collection.insert([{"code": "FR", "name": "France"}])
+        >>> for r in db.iter_validate_database():
+        ...    print(r.message[0:32])
+        'capital' is a required property
+        'continent' is a required proper
+
         :param kwargs:
         :return: iterator over validation results
         """
         for collection in self.list_collections():
             yield from collection.iter_validate_collection(**kwargs)
+        if self.metadata.ensure_referential_integrity:
+            yield from self._validate_referential_integrity(**kwargs)
+
+    def _validate_referential_integrity(self, **kwargs) -> Iterator["ValidationResult"]:
+        """
+        Validate referential integrity of the database.
+
+        :param kwargs:
+        :return: iterator over validation results
+        """
+        sv = self.schema_view
+        cmap = defaultdict(list)
+        for collection in self.list_collections():
+            if not collection.target_class_name:
+                raise ValueError(f"Collection {collection.name} has no target class")
+            cmap[collection.target_class_name].append(collection)
+        for collection in self.list_collections():
+            cd = collection.class_definition()
+            induced_slots = sv.class_induced_slots(cd.name)
+            slot_map = {s.name: s for s in induced_slots}
+            # rmap = {s.name: s.range for s in induced_slots}
+            sr_to_coll = {s.name: cmap.get(s.range, []) for s in induced_slots if s.range}
+            for obj in collection.find_iter():
+                for k, v in obj.items():
+                    if k not in sr_to_coll:
+                        continue
+                    ref_colls = sr_to_coll[k]
+                    if not ref_colls:
+                        continue
+                    if not isinstance(v, (str, int)):
+                        continue
+                    slot = slot_map[k]
+                    found = False
+                    for ref_coll in ref_colls:
+                        ref_obj = ref_coll.get_one(v)
+                        if ref_obj:
+                            found = True
+                            break
+                    if not found:
+                        yield ValidationResult(
+                            type="ReferentialIntegrity",
+                            severity=Severity.ERROR,
+                            message=f"Referential integrity error: {slot.range} not found",
+                            instantiates=slot.range,
+                            instance=v,
+                        )
 
     def drop(self, **kwargs):
         """
-        Drop the database and all collections
+        Drop the database and all collections.
+
+        :param kwargs: additional arguments
         """
         raise NotImplementedError()
+
+    def import_database(self, location: str, source_format: Optional[str] = None, **kwargs):
+        """
+        Import a database from a file or location.
+
+        :param location: location of the file
+        :param source_format: source format
+        :param kwargs: additional arguments
+        """
+        objects = load_objects(location, format=source_format)
+        for obj in objects:
+            self.store(obj)
+
+    def export_database(self, location: str, target_format: Optional[str] = None, **kwargs):
+        """
+        Export a database to a file or location.
+
+        :param location: location of the file
+        :param target_format: target format
+        :param kwargs: additional arguments
+        """
+        obj = {}
+        for coll in self.list_collections():
+            qr = coll.find({}, limit=-1)
+            obj[coll.alias] = qr.rows
+        logger.info(f"Exporting object with {len(obj)} collections to {location} in {target_format} format")
+        with open(location, "w", encoding="utf-8") as stream:
+            stream.write(render_output(obj, format=target_format))

linkml_store/api/stores/chromadb/__init__.py

@@ -0,0 +1,7 @@
+"""
+Adapter for ChromaDB vector database.
+
+.. warning::
+
+    Support for ChromaDB is experimental and may change in the future.
+"""

linkml_store/api/stores/chromadb/chromadb_collection.py

@@ -1,3 +1,7 @@
+"""
+ChromaDB Collection
+"""
+
 import logging
 from typing import Any, Dict, List, Optional, Tuple, Union
 
@@ -13,6 +17,9 @@ logger = logging.getLogger(__name__)
 
 
 class ChromaDBCollection(Collection):
+    """
+    A wrapper for ChromaDB collections.
+    """
 
     @property
     def native_collection(self) -> ChromaCollection:
@@ -50,7 +57,7 @@ class ChromaDBCollection(Collection):
         return len(ids)
 
     def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> int:
-        logger.info(f"Deleting from {self._target_class_name} where: {where}")
+        logger.info(f"Deleting from {self.target_class_name} where: {where}")
         if where is None:
             where = {}
         results = self.native_collection.get(where=where)

linkml_store/api/stores/duckdb/__init__.py

@@ -0,0 +1,16 @@
+"""
+Adapter for DuckDB embedded database.
+
+Handles have the form:
+
+ - ``duckdb:///<path>`` for a file-based database
+ - ``duckdb:///:memory:`` for an in-memory database
+"""
+
+from linkml_store.api.stores.duckdb.duckdb_collection import DuckDBCollection
+from linkml_store.api.stores.duckdb.duckdb_database import DuckDBDatabase
+
+__all__ = [
+    "DuckDBCollection",
+    "DuckDBDatabase",
+]

linkml_store/api/stores/duckdb/duckdb_collection.py

@@ -19,16 +19,18 @@ class DuckDBCollection(Collection):
     _table_created: bool = None
 
     def insert(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
+        logger.debug(f"Inserting {len(objs)}")
         if not isinstance(objs, list):
             objs = [objs]
         if not objs:
             return
         cd = self.class_definition()
         if not cd:
+            logger.debug(f"No class definition defined for {self.alias} {self.target_class_name}; will induce")
             cd = self.induce_class_definition_from_objects(objs)
         self._create_table(cd)
         table = self._sqla_table(cd)
-        logger.info(f"Inserting into: {self._alias} // T={table.name}")
+        logger.info(f"Inserting into: {self.alias} // T={table.name}")
         engine = self.parent.engine
         col_names = [c.name for c in table.columns]
         objs = [{k: obj.get(k, None) for k in col_names} for obj in objs]
@@ -37,7 +39,7 @@ class DuckDBCollection(Collection):
                 conn.execute(insert(table), objs)
             conn.commit()
 
-    def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> int:
+    def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> Optional[int]:
         if not isinstance(objs, list):
             objs = [objs]
         cd = self.class_definition()
@@ -52,15 +54,15 @@ class DuckDBCollection(Collection):
                 stmt = stmt.compile(engine)
                 conn.execute(stmt)
                 conn.commit()
-        return len(objs)
+        return
 
-    def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> int:
-        logger.info(f"Deleting from {self._target_class_name} where: {where}")
+    def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> Optional[int]:
+        logger.info(f"Deleting from {self.target_class_name} where: {where}")
         if where is None:
             where = {}
         cd = self.class_definition()
         if not cd:
-            logger.info(f"No class definition found for {self._target_class_name}, assuming not prepopulated")
+            logger.info(f"No class definition found for {self.target_class_name}, assuming not prepopulated")
             return 0
         table = self._sqla_table(cd)
         engine = self.parent.engine
@@ -78,7 +80,7 @@ class DuckDBCollection(Collection):
             if deleted_rows_count == 0 and not missing_ok:
                 raise ValueError(f"No rows found for {where}")
             conn.commit()
-            return deleted_rows_count
+            return deleted_rows_count if deleted_rows_count > -1 else None
 
     def query_facets(
         self, where: Dict = None, facet_columns: List[str] = None, facet_limit=DEFAULT_FACET_LIMIT, **kwargs
@@ -115,7 +117,7 @@ class DuckDBCollection(Collection):
                 typ = sqla.ARRAY(typ, dimensions=1)
             col = Column(att.name, typ)
             cols.append(col)
-        t = Table(self._alias, metadata_obj, *cols)
+        t = Table(self.alias, metadata_obj, *cols)
         return t
 
     def _create_table(self, cd: ClassDefinition):
@@ -123,7 +125,7 @@ class DuckDBCollection(Collection):
             logger.info(f"Already have table for: {cd.name}")
             return
         query = Query(
-            from_table="information_schema.tables", where_clause={"table_type": "BASE TABLE", "table_name": self._alias}
+            from_table="information_schema.tables", where_clause={"table_type": "BASE TABLE", "table_name": self.alias}
         )
         qr = self.parent.query(query)
         if qr.num_rows > 0: