linkml-store 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

linkml_store/api/database.py

@@ -3,7 +3,24 @@ from abc import ABC
 from collections import defaultdict
 from copy import copy
 from pathlib import Path
-from typing import TYPE_CHECKING, ClassVar, Dict, Iterator, Optional, Sequence, Type, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    ClassVar,
+    Dict,
+    Generic,
+    Iterator,
+    List,
+    Optional,
+    Sequence,
+    Type,
+    Union,
+)
+
+from linkml_store.api.types import CollectionType
+from linkml_store.utils.format_utils import load_objects, render_output
+from linkml_store.utils.patch_utils import PatchDict
 
 try:
     from linkml.validator.report import Severity, ValidationResult
@@ -22,18 +39,40 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+LISTENER = Callable[[Collection, List[PatchDict]], None]
 
-class Database(ABC):
+
+class Database(ABC, Generic[CollectionType]):
     """
     A Database provides access to named collections of data.
 
-    Examples
-    --------
+    A database object is owned by a :ref:`Client`. The database
+    object uses a :ref:`handle` to know what kind of external
+    dataase system to connect to (e.g. duckdb, mongodb). The handle
+    is a string ``<DatabaseType>:<LocalLocator>``
+
+    The
+    database object may also have an :ref:`alias` that is mapped
+    to the handle.
+
+    Attaching a database
+    --------------------
     >>> from linkml_store.api.client import Client
     >>> client = Client()
-    >>> db = client.attach_database("duckdb", alias="test")
+    >>> db = client.attach_database("duckdb:///:memory:", alias="test")
+
+    We can check the value of the handle:
+
     >>> db.handle
     'duckdb:///:memory:'
+
+    The alias can be used to retrieve the database object from the client
+
+    >>> assert db == client.get_database("test")
+
+    Creating a collection
+    ---------------------
+
     >>> collection = db.create_collection("Person")
     >>> len(db.list_collections())
     1
@@ -57,11 +96,18 @@ class Database(ABC):
     """
 
     _schema_view: Optional[SchemaView] = None
+    """Schema for the database. May be transformed."""
+
+    _original_schema_view: Optional[SchemaView] = None
+    """If a schema must be transformed, then the original is stored here."""
+
     _collections: Optional[Dict[str, Collection]] = None
     parent: Optional["Client"] = None
     metadata: Optional[DatabaseConfig] = None
     collection_class: ClassVar[Optional[Type[Collection]]] = None
 
+    listeners: Optional[List[LISTENER]] = None
+
     def __init__(self, handle: Optional[str] = None, metadata: Optional[DatabaseConfig] = None, **kwargs):
         if metadata:
             self.metadata = metadata
@@ -101,6 +147,8 @@ class Database(ABC):
         return self
 
     def _initialize_collections(self):
+        if not self.metadata.collections:
+            return
         for name, collection_config in self.metadata.collections.items():
             alias = collection_config.alias
             typ = collection_config.type
@@ -127,15 +175,46 @@ class Database(ABC):
 
     @property
     def recreate_if_exists(self) -> bool:
+        """
+        Return whether to recreate the database if it already exists.
+
+        :return:
+        """
         return self.metadata.recreate_if_exists
 
     @property
     def handle(self) -> str:
+        """
+        Return the database handle.
+
+        Examples:
+
+        - ``duckdb:///:memory:``
+        - ``duckdb:///tmp/test.db``
+        - ``mongodb://localhost:27017/``
+
+        :return:
+        """
         return self.metadata.handle
 
-    def store(self, obj: Dict[str, str], **kwargs):
+    @property
+    def alias(self):
+        return self.metadata.alias
+
+    def store(self, obj: Dict[str, Any], **kwargs):
         """
-        Store an object in the database
+        Store an object in the database.
+
+        The object is assumed to be a Dictionary of Collections.
+
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> db.store({"persons": [{"id": "P1", "name": "John", "age_in_years": 30}]})
+        >>> collection = db.get_collection("persons")
+        >>> qr = collection.find()
+        >>> qr.num_rows
+        1
 
         :param obj: object to store
         :param kwargs: additional arguments
@@ -144,6 +223,7 @@ class Database(ABC):
         roots = [c for c in sv.all_classes().values() if c.tree_root]
         root = roots[0] if roots else None
         for k, v in obj.items():
+            logger.info(f"Storing collection {k}")
             if root:
                 slot = sv.induced_slot(k, root.name)
                 if not slot:
@@ -158,20 +238,29 @@ class Database(ABC):
             if not v:
                 continue
             if slot:
-                collection = self.get_collection(slot.range, create_if_not_exists=True)
+                logger.debug(f"Aligning to existing slot: {slot.name} range={slot.range}")
+                collection = self.get_collection(slot.name, type=slot.range, create_if_not_exists=True)
             else:
                 collection = self.get_collection(k, create_if_not_exists=True)
+            logger.debug(f"Replacing using {collection.alias} {collection.target_class_name}")
             collection.replace(v)
 
     def commit(self, **kwargs):
         """
-        Commit any pending changes to the database
+        Commit pending changes to the database.
+
+        :param kwargs:
+        :return:
         """
-        raise NotImplementedError()
+        for coll in self.list_collections():
+            coll.commit()
 
     def close(self, **kwargs):
         """
-        Close the database and all connection objects
+        Close the database.
+
+        :param kwargs:
+        :return:
         """
         raise NotImplementedError()
 
@@ -188,15 +277,27 @@ class Database(ABC):
         **kwargs,
     ) -> Collection:
         """
-        Create a new collection
+        Create a new collection in the current database.
+
+        The collection must have a *Type*, and may have an *Alias*.
+
+        Examples:
 
         >>> from linkml_store.api.client import Client
         >>> client = Client()
         >>> db = client.attach_database("duckdb", alias="test")
-        >>> collection = db.create_collection("Person")
-        >>> collection.name
+        >>> collection = db.create_collection("Person", alias="persons")
+        >>> collection.alias
+        'persons'
+        >>> collection.target_class_name
         'Person'
 
+        If alias is not provided, it defaults to the name of the type.
+
+        >>> collection = db.create_collection("Organization")
+        >>> collection.alias
+        'Organization'
+
         :param name: name of the collection
         :param alias: alias for the collection
         :param metadata: metadata for the collection
@@ -207,6 +308,8 @@ class Database(ABC):
             raise ValueError(f"Collection name must be provided: alias: {alias} metadata: {metadata}")
         collection_cls = self.collection_class
         collection = collection_cls(name=name, alias=alias, parent=self, metadata=metadata)
+        if metadata and metadata.source_location:
+            collection.load_from_source()
         if metadata and metadata.attributes:
             sv = self.schema_view
             schema = sv.schema
@@ -218,6 +321,7 @@ class Database(ABC):
             alias = name
         self._collections[alias] = collection
         if recreate_if_exists:
+            logger.debug(f"Recreating collection {collection.name}")
             collection.delete_where({}, missing_ok=True)
         return collection
 
@@ -265,7 +369,9 @@ class Database(ABC):
         """
         return [c.name for c in self.list_collections(**kwargs)]
 
-    def get_collection(self, name: str, create_if_not_exists=True, **kwargs) -> "Collection":
+    def get_collection(
+        self, name: str, type: Optional[str] = None, create_if_not_exists=True, **kwargs
+    ) -> "Collection":
         """
         Get a named collection.
 
@@ -283,14 +389,19 @@ class Database(ABC):
         KeyError: 'Collection NonExistent does not exist'
 
         :param name: name of the collection
+        :param type: target class name
         :param create_if_not_exists: create the collection if it does not exist
 
         """
         if not self._collections:
+            logger.debug("Initializing collections")
             self.init_collections()
         if name not in self._collections.keys():
             if create_if_not_exists:
-                self._collections[name] = self.create_collection(name)
+                if type is None:
+                    type = name
+                logger.debug(f"Creating new collection: {name} kwargs: {kwargs}")
+                self._collections[name] = self.create_collection(type, alias=name, **kwargs)
             else:
                 raise KeyError(f"Collection {name} does not exist")
         return self._collections[name]
@@ -328,12 +439,38 @@ class Database(ABC):
         :return:
 
         """
-        raise NotImplementedError
+        if query.from_table:
+            collection = self.get_collection(query.from_table)
+            return collection.query(query, **kwargs)
+        else:
+            raise NotImplementedError(f"Querying without a table is not supported in {self.__class__.__name__}")
 
     @property
     def schema_view(self) -> SchemaView:
         """
-        Return a schema view for the named collection
+        Return a schema view for the named collection.
+
+        If no explicit schema is provided, this will generalize one
+
+        Induced schema example:
+
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person", alias="persons")
+        >>> collection.insert([{"id": "P1", "name": "John", "age_in_years": 25}])
+        >>> schema_view = db.schema_view
+        >>> cd = schema_view.get_class("Person")
+        >>> cd.attributes["id"].range
+        'string'
+        >>> cd.attributes["age_in_years"].range
+        'integer'
+
+        We can reuse the same class:
+
+        >>> collection2 = db.create_collection("Person", alias="other_persons")
+        >>> collection2.class_definition().attributes["age_in_years"].range
+        'integer'
         """
         if not self._schema_view:
             self._initialize_schema()
@@ -345,6 +482,26 @@ class Database(ABC):
         """
         Set the schema view for the database.
 
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> sv = SchemaView("tests/input/countries/countries.linkml.yaml")
+        >>> db.set_schema_view(sv)
+        >>> cd = db.schema_view.schema.classes["Country"]
+        >>> sorted(cd.slots)
+        ['capital', 'code', 'continent', 'languages', 'name']
+        >>> induced_slots = {s.name: s for s in sv.class_induced_slots("Country")}
+        >>> sorted(induced_slots.keys())
+        ['capital', 'code', 'continent', 'languages', 'name']
+        >>> induced_slots["code"].identifier
+        True
+
+        Creating a new collection will align with the schema view:
+
+        >>> collection = db.create_collection("Country", "all_countries")
+        >>> sorted(collection.class_definition().slots)
+        ['capital', 'code', 'continent', 'languages', 'name']
+
         :param schema_view:
         :return:
         """
@@ -375,8 +532,7 @@ class Database(ABC):
                 if inlined and slot.range:
                     if slot.name in self._collections:
                         coll = self._collections[slot.name]
-                        if not coll.metadata.type:
-                            coll.metadata.type = slot.range
+                        coll.metadata.type = slot.range
 
     def load_schema_view(self, path: Union[str, Path]):
         """
@@ -386,6 +542,21 @@ class Database(ABC):
         >>> client = Client()
         >>> db = client.attach_database("duckdb", alias="test")
         >>> db.load_schema_view("tests/input/countries/countries.linkml.yaml")
+        >>> sv = db.schema_view
+        >>> cd = sv.schema.classes["Country"]
+        >>> sorted(cd.slots)
+        ['capital', 'code', 'continent', 'languages', 'name']
+        >>> induced_slots = {s.name: s for s in sv.class_induced_slots("Country")}
+        >>> sorted(induced_slots.keys())
+        ['capital', 'code', 'continent', 'languages', 'name']
+        >>> induced_slots["code"].identifier
+        True
+
+        Creating a new collection will align with the schema view:
+
+        >>> collection = db.create_collection("Country", "all_countries")
+        >>> sorted(collection.class_definition().slots)
+        ['capital', 'code', 'continent', 'languages', 'name']
 
         :param path:
         :return:
@@ -420,6 +591,42 @@ class Database(ABC):
         """
         Validate the contents of the database.
 
+        An an example, let's create a database with a predefined schema
+        from the countries.linkml.yaml file:
+
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> db.load_schema_view("tests/input/countries/countries.linkml.yaml")
+
+        Let's introspect the schema to see what slots are applicable for the class "Country":
+
+        >>> sv = db.schema_view
+        >>> for slot in sv.class_induced_slots("Country"):
+        ...     print(slot.name, slot.range, slot.required)
+        name string True
+        code string True
+        capital string True
+        continent string True
+        languages Language None
+
+        Next we'll create a collection, binding it to the target class "Country", and insert
+        valid data:
+
+        >>> collection = db.create_collection("Country", "all_countries")
+        >>> obj = {"code": "US", "name": "United States", "continent": "North America", "capital": "Washington, D.C."}
+        >>> collection.insert([obj])
+        >>> list(db.iter_validate_database())
+        []
+
+        Now let's insert some invalid data (missing required fields)
+
+        >>> collection.insert([{"code": "FR", "name": "France"}])
+        >>> for r in db.iter_validate_database():
+        ...    print(r.message[0:32])
+        'capital' is a required property
+        'continent' is a required proper
+
         :param kwargs:
         :return: iterator over validation results
         """
@@ -474,6 +681,42 @@ class Database(ABC):
 
     def drop(self, **kwargs):
         """
-        Drop the database and all collections
+        Drop the database and all collections.
+
+        :param kwargs: additional arguments
         """
         raise NotImplementedError()
+
+    def import_database(self, location: str, source_format: Optional[str] = None, **kwargs):
+        """
+        Import a database from a file or location.
+
+        :param location: location of the file
+        :param source_format: source format
+        :param kwargs: additional arguments
+        """
+        objects = load_objects(location, format=source_format)
+        for obj in objects:
+            self.store(obj)
+
+    def export_database(self, location: str, target_format: Optional[str] = None, **kwargs):
+        """
+        Export a database to a file or location.
+
+        :param location: location of the file
+        :param target_format: target format
+        :param kwargs: additional arguments
+        """
+        obj = {}
+        for coll in self.list_collections():
+            qr = coll.find({}, limit=-1)
+            obj[coll.alias] = qr.rows
+        logger.info(f"Exporting object with {len(obj)} collections to {location} in {target_format} format")
+        with open(location, "w", encoding="utf-8") as stream:
+            stream.write(render_output(obj, format=target_format))
+
+    def broadcast(self, source: Collection, patches: List[PatchDict]):
+        if not self.listeners:
+            return
+        for listener in self.listeners:
+            listener(source, patches)

linkml_store/api/stores/chromadb/__init__.py

@@ -1,3 +1,7 @@
 """
-Support for ChromaDB is experimental.
+Adapter for ChromaDB vector database.
+
+.. warning::
+
+    Support for ChromaDB is experimental and may change in the future.
 """

linkml_store/api/stores/duckdb/__init__.py

@@ -1,3 +1,12 @@
+"""
+Adapter for DuckDB embedded database.
+
+Handles have the form:
+
+ - ``duckdb:///<path>`` for a file-based database
+ - ``duckdb:///:memory:`` for an in-memory database
+"""
+
 from linkml_store.api.stores.duckdb.duckdb_collection import DuckDBCollection
 from linkml_store.api.stores.duckdb.duckdb_database import DuckDBDatabase
 

linkml_store/api/stores/duckdb/duckdb_collection.py

@@ -19,12 +19,14 @@ class DuckDBCollection(Collection):
     _table_created: bool = None
 
     def insert(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
+        logger.debug(f"Inserting {len(objs)}")
         if not isinstance(objs, list):
             objs = [objs]
         if not objs:
             return
         cd = self.class_definition()
         if not cd:
+            logger.debug(f"No class definition defined for {self.alias} {self.target_class_name}; will induce")
             cd = self.induce_class_definition_from_objects(objs)
         self._create_table(cd)
         table = self._sqla_table(cd)
@@ -36,8 +38,9 @@ class DuckDBCollection(Collection):
             with conn.begin():
                 conn.execute(insert(table), objs)
             conn.commit()
+        self._post_insert_hook(objs)
 
-    def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> int:
+    def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> Optional[int]:
         if not isinstance(objs, list):
             objs = [objs]
         cd = self.class_definition()
@@ -52,9 +55,9 @@ class DuckDBCollection(Collection):
                 stmt = stmt.compile(engine)
                 conn.execute(stmt)
                 conn.commit()
-        return len(objs)
+        return
 
-    def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> int:
+    def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> Optional[int]:
         logger.info(f"Deleting from {self.target_class_name} where: {where}")
         if where is None:
             where = {}
@@ -78,7 +81,7 @@ class DuckDBCollection(Collection):
             if deleted_rows_count == 0 and not missing_ok:
                 raise ValueError(f"No rows found for {where}")
             conn.commit()
-            return deleted_rows_count
+            return deleted_rows_count if deleted_rows_count > -1 else None
 
     def query_facets(
         self, where: Dict = None, facet_columns: List[str] = None, facet_limit=DEFAULT_FACET_LIMIT, **kwargs

linkml_store/api/stores/duckdb/duckdb_database.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from pathlib import Path
 from typing import Optional
 
 import pandas as pd
@@ -22,6 +23,7 @@ TYPE_MAP = {
     "DATE": "date",
     "DOUBLE": "float",
     "INTEGER": "integer",
+    "JSON": "Any",
 }
 
 
@@ -33,9 +35,13 @@ class DuckDBDatabase(Database):
     _engine: sqlalchemy.Engine = None
     collection_class = DuckDBCollection
 
-    def __init__(self, handle: Optional[str] = None, **kwargs):
+    def __init__(self, handle: Optional[str] = None, recreate_if_exists: bool = False, **kwargs):
         if handle is None:
             handle = "duckdb:///:memory:"
+        if recreate_if_exists:
+            path = Path(handle.replace("duckdb:///", ""))
+            if path.exists():
+                path.unlink()
         super().__init__(handle=handle, **kwargs)
 
     @property
@@ -69,7 +75,10 @@ class DuckDBDatabase(Database):
                 if qr.num_rows == 0:
                     logger.debug(f"Table {query.from_table} not created yet")
                     return QueryResult(query=query, num_rows=0, rows=[])
-            sv = self._schema_view
+            if not query.from_table.startswith("information_schema"):
+                sv = self.schema_view
+            else:
+                sv = None
             if sv:
                 cd = None
                 for c in self._collections.values():
@@ -107,7 +116,10 @@ class DuckDBDatabase(Database):
 
     def init_collections(self):
         # TODO: unify schema introspection
-        schema = introspect_schema(self.engine)
+        if not self.schema_view:
+            schema = introspect_schema(self.engine)
+        else:
+            schema = self.schema_view.schema
         table_names = schema.classes.keys()
         if self._collections is None:
             self._collections = {}
@@ -119,7 +131,7 @@ class DuckDBDatabase(Database):
     def induce_schema_view(self) -> SchemaView:
         # TODO: unify schema introspection
         # TODO: handle case where schema is provided in advance
-        logger.info(f"Inducing schema view for {self.metadata.handle}")
+        logger.info(f"Inducing schema view for {self.metadata.handle} // {self}")
         sb = SchemaBuilder()
         schema = sb.schema
         query = Query(from_table="information_schema.tables", where_clause={"table_type": "BASE TABLE"})
@@ -144,8 +156,10 @@ class DuckDBDatabase(Database):
             sd = SlotDefinition(
                 row["column_name"], required=row["is_nullable"] == "NO", multivalued=multivalued, range=rng
             )
+            if dt == "JSON":
+                sd.inlined_as_list = True
             sb.schema.classes[tbl_name].attributes[sd.name] = sd
-            logger.info(f"Introspected slot: {tbl_name}.{sd.name}: {sd.range}")
+            logger.info(f"Introspected slot: {tbl_name}.{sd.name}: {sd.range} FROM {dt}")
         sb.add_defaults()
         for cls_name in schema.classes:
             if cls_name in self.metadata.collections:

linkml_store/api/stores/duckdb/mappings.py

@@ -3,5 +3,6 @@ import sqlalchemy as sqla
 TMAP = {
     "string": sqla.String,
     "integer": sqla.Integer,
+    "float": sqla.Float,
     "linkml:Any": sqla.JSON,
 }

linkml_store/api/stores/filesystem/__init__.py

@@ -0,0 +1,15 @@
+"""
+Adapter for FileSystem wrapper
+
+Handles have the form:
+
+ - ``file:<path>`` for a local file
+ """
+
+from linkml_store.api.stores.filesystem.filesystem_collection import FileSystemCollection
+from linkml_store.api.stores.filesystem.filesystem_database import FileSystemDatabase
+
+__all__ = [
+    "FileSystemCollection",
+    "FileSystemDatabase",
+]