linkml-store 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

linkml_store/api/client.py

@@ -1,3 +1,4 @@
+import logging
 from pathlib import Path
 from typing import Dict, Optional, Union
 
@@ -8,14 +9,19 @@ from linkml_store.api import Database
 from linkml_store.api.config import ClientConfig
 from linkml_store.api.stores.chromadb.chromadb_database import ChromaDBDatabase
 from linkml_store.api.stores.duckdb.duckdb_database import DuckDBDatabase
+from linkml_store.api.stores.filesystem.filesystem_database import FileSystemDatabase
 from linkml_store.api.stores.mongodb.mongodb_database import MongoDBDatabase
 from linkml_store.api.stores.solr.solr_database import SolrDatabase
 
+logger = logging.getLogger(__name__)
+
+
 HANDLE_MAP = {
     "duckdb": DuckDBDatabase,
     "solr": SolrDatabase,
     "mongodb": MongoDBDatabase,
     "chromadb": ChromaDBDatabase,
+    "file": FileSystemDatabase,
 }
 
 
@@ -23,14 +29,27 @@ class Client:
     """
     A client is the top-level object for interacting with databases.
 
-    A client has access to one or more :class:`Database` objects.
-
-    Each database consists of a number of :class:`.Collection` objects.
+    * A client has access to one or more :class:`.Database` objects.
+    * Each database consists of a number of :class:`.Collection` objects.
 
-    Examples
-    --------
+    Creating a client
+    -----------------
     >>> client = Client()
+
+    Attaching a database
+    --------------------
     >>> db = client.attach_database("duckdb", alias="test")
+
+    Note that normally a handle would be specified by a locator such as ``duckdb:///<PATH>``, but
+    for convenience, an in-memory duckdb object can be specified without a full locator
+
+    We can check the actual handle:
+
+    >>> db.handle
+    'duckdb:///:memory:'
+
+    Creating a new collection
+    -------------------------
     >>> collection = db.create_collection("Person")
     >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
     >>> collection.insert(objs)
@@ -151,6 +170,8 @@ class Client:
         if ":" not in handle:
             scheme = handle
             handle = None
+            if alias is None:
+                alias = scheme
         else:
             scheme, _ = handle.split(":", 1)
         if scheme not in HANDLE_MAP:
@@ -165,6 +186,11 @@ class Client:
             self._databases = {}
         self._databases[alias] = db
         db.parent = self
+        if db.alias:
+            if db.alias != alias:
+                raise AssertionError(f"Inconsistent alias: {db.alias} != {alias}")
+        else:
+            db.metadata.alias = alias
         return db
 
     def get_database(self, name: Optional[str] = None, create_if_not_exists=True, **kwargs) -> Database:
@@ -195,6 +221,7 @@ class Client:
             self._databases = {}
         if name not in self._databases:
             if create_if_not_exists:
+                logger.info(f"Creating database: {name}")
                 self.attach_database(name, **kwargs)
             else:
                 raise ValueError(f"Database {name} does not exist")

linkml_store/api/collection.py

@@ -1,16 +1,22 @@
+"""A structure for representing collections of similar objects."""
+
 import hashlib
 import logging
 from collections import defaultdict
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, TextIO, Type, Union
+from typing import TYPE_CHECKING, Any, Dict, Generic, Iterator, List, Optional, TextIO, Tuple, Type, Union
 
 import numpy as np
+from linkml_runtime import SchemaView
 from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition
 from linkml_runtime.linkml_model.meta import ArrayExpression
 from pydantic import BaseModel
 
+from linkml_store.api.types import DatabaseType
 from linkml_store.index import get_indexer
+from linkml_store.utils.format_utils import load_objects
 from linkml_store.utils.object_utils import clean_empties
+from linkml_store.utils.patch_utils import PatchDict, apply_patches_to_list, patches_from_objects_lists
 
 try:
     from linkml.validator.report import ValidationResult
@@ -33,17 +39,27 @@ IDENTIFIER = str
 FIELD_NAME = str
 
 
-class Collection:
+class Collection(Generic[DatabaseType]):
     """
     A collection is an organized set of objects of the same or similar type.
 
     - For relational databases, a collection is typically a table
     - For document databases such as MongoDB, a collection is the native type
-    - For a file system, a collection could be a single tabular file such as Parquet or CSV
+    - For a file system, a collection could be a single tabular file such as Parquet or CSV.
+
+    Collection objects are typically not created directly - instead they are generated
+    from a parent :class:`.Database` object:
+
+    >>> from linkml_store import Client
+    >>> client = Client()
+    >>> db = client.attach_database("duckdb", alias="test")
+    >>> collection = db.create_collection("Person")
+    >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+    >>> collection.insert(objs)
     """
 
     # name: str
-    parent: Optional["Database"] = None
+    parent: Optional[DatabaseType] = None
     _indexers: Optional[Dict[str, Indexer]] = None
     # hidden: Optional[bool] = False
 
@@ -57,15 +73,21 @@ class Collection:
             self.metadata = metadata
         else:
             self.metadata = CollectionConfig(name=name, **kwargs)
-        if name is not None and self.metadata.name is not None and name != self.metadata.name:
-            raise ValueError(f"Name mismatch: {name} != {self.metadata.name}")
+            if not self.metadata.alias:
+                self.metadata.alias = name
+            if not self.metadata.type:
+                self.metadata.type = name
+        # if name is not None and self.metadata.name is not None and name != self.metadata.name:
+        #    raise ValueError(f"Name mismatch: {name} != {self.metadata.name}")
 
     @property
     def name(self) -> str:
         """
-        Return the name of the collection
+        Return the name of the collection.
 
-        :return:
+        TODO: deprecate in favor of Type
+
+        :return: name of the collection
         """
         return self.metadata.name
 
@@ -79,7 +101,7 @@ class Collection:
 
         :return: True if the collection is hidden
         """
-        return self.metadata.hidden
+        # return self.metadata.hidden
 
     @property
     def target_class_name(self):
@@ -88,7 +110,14 @@ class Collection:
 
         This MUST be a LinkML class name
 
-        :return:
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person", alias="persons")
+        >>> collection.target_class_name
+        'Person'
+
+        :return: name of the class which members of this collection instantiate
         """
         # TODO: this is a shim layer until we can normalize on this
         if self.metadata.type:
@@ -104,15 +133,34 @@ class Collection:
         to have an alias, for example "persons" which collects all instances
         of class Person.
 
-        The _alias SHOULD be used for Table names in SQL.
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person", alias="persons")
+        >>> collection.alias
+        'persons'
+
+        If no explicit alias is provided, then the target class name is used:
+
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person")
+        >>> collection.alias
+        'Person'
+
+        The alias SHOULD be used for Table names in SQL.
 
         For nested data, the alias SHOULD be used as the key; e.g
 
-         ``{ "persons": [ { "name": "Alice" }, { "name": "Bob" } ] }``
+        .. code-block:: json
+
+           { "persons": [ { "name": "Alice" }, { "name": "Bob" } ] }
 
         :return:
         """
         # TODO: this is a shim layer until we can normalize on this
+        # TODO: this is a shim layer until we can normalize on this
         if self.metadata.alias:
             return self.metadata.alias
         return self.name
@@ -121,6 +169,13 @@ class Collection:
         """
         Replace entire collection with objects.
 
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person")
+        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+        >>> collection.insert(objs)
+
         :param objs:
         :param kwargs:
         :return:
@@ -130,7 +185,14 @@ class Collection:
 
     def insert(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         """
-        Add one or more objects to the collection
+        Add one or more objects to the collection.
+
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person")
+        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+        >>> collection.insert(objs)
 
         :param objs:
         :param kwargs:
@@ -138,9 +200,36 @@ class Collection:
         """
         raise NotImplementedError
 
-    def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> int:
+    def _post_insert_hook(self, objs: List[OBJECT], **kwargs):
+        patches = [{"op": "add", "path": "/0", "value": obj} for obj in objs]
+        self._broadcast(patches, **kwargs)
+
+    def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> Optional[int]:
         """
-        Delete one or more objects from the collection
+        Delete one or more objects from the collection.
+
+        First let's set up a collection:
+
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person")
+        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+        >>> collection.insert(objs)
+        >>> collection.find({}).num_rows
+        2
+
+        Now let's delete an object:
+
+        >>> collection.delete(objs[0])
+        >>> collection.find({}).num_rows
+        1
+
+        Deleting the same object again should have no effect:
+
+        >>> collection.delete(objs[0])
+        >>> collection.find({}).num_rows
+        1
 
         :param objs:
         :param kwargs:
@@ -148,9 +237,30 @@ class Collection:
         """
         raise NotImplementedError
 
-    def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> int:
+    def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> Optional[int]:
         """
-        Delete objects that match a query
+        Delete objects that match a query.
+
+        First let's set up a collection:
+
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person")
+        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+        >>> collection.insert(objs)
+
+        Now let's delete an object:
+
+        >>> collection.delete_where({"id": "P1"})
+        >>> collection.find({}).num_rows
+        1
+
+        Match everything:
+
+        >>> collection.delete_where({})
+        >>> collection.find({}).num_rows
+        0
 
         :param where: where conditions
         :param missing_ok: if True, do not raise an error if the collection does not exist
@@ -161,7 +271,7 @@ class Collection:
 
     def update(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         """
-        Update one or more objects in the collection
+        Update one or more objects in the collection.
 
         :param objs:
         :param kwargs:
@@ -174,7 +284,21 @@ class Collection:
 
     def query(self, query: Query, **kwargs) -> QueryResult:
         """
-        Run a query against the collection
+        Run a query against the collection.
+
+        First let's load a collection:
+
+        >>> from linkml_store import Client
+        >>> from linkml_store.utils.format_utils import load_objects
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb")
+        >>> collection = db.create_collection("Country")
+        >>> objs = load_objects("tests/input/countries/countries.jsonl")
+        >>> collection.insert(objs)
+
+        Now let's run a query:
+
+        TODO
 
         :param query:
         :param kwargs:
@@ -184,7 +308,7 @@ class Collection:
 
     def query_facets(
         self, where: Optional[Dict] = None, facet_columns: List[str] = None, facet_limit=DEFAULT_FACET_LIMIT, **kwargs
-    ) -> Dict[str, Dict[str, int]]:
+    ) -> Dict[str, List[Tuple[Any, int]]]:
         """
         Run a query to get facet counts for one or more columns.
 
@@ -202,12 +326,12 @@ class Collection:
         :param query: A Query object representing the base query.
         :param facet_columns: A list of column names to get facet counts for.
         :param facet_limit:
-        :return: A dictionary where keys are column names and values are pandas DataFrames
+        :return: A dictionary where keys are column names and values are tuples
                  containing the facet counts for each unique value in the respective column.
         """
         raise NotImplementedError
 
-    def get(self, ids: Optional[IDENTIFIER], **kwargs) -> QueryResult:
+    def get(self, ids: Optional[List[IDENTIFIER]], **kwargs) -> QueryResult:
         """
         Get one or more objects by ID.
 
@@ -217,6 +341,8 @@ class Collection:
         """
         # TODO
         id_field = self.identifier_attribute_name
+        if not id_field:
+            raise ValueError(f"No identifier for {self.name}")
         return self.find({id_field: ids})
 
     def get_one(self, id: IDENTIFIER, **kwargs) -> Optional[OBJECT]:
@@ -242,6 +368,31 @@ class Collection:
         """
         Find objects in the collection using a where query.
 
+        As an example, first load a collection:
+
+        >>> from linkml_store import Client
+        >>> from linkml_store.utils.format_utils import load_objects
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb")
+        >>> collection = db.create_collection("Country")
+        >>> objs = load_objects("tests/input/countries/countries.jsonl")
+        >>> collection.insert(objs)
+
+        Now let's find all objects:
+
+        >>> qr = collection.find({})
+        >>> qr.num_rows
+        20
+
+        We can do a more restrictive query:
+
+        >>> qr = collection.find({"code": "FR"})
+        >>> qr.num_rows
+        1
+        >>> qr.rows[0]["name"]
+        'France'
+
+
         :param where:
         :param kwargs:
         :return:
@@ -290,6 +441,7 @@ class Collection:
             raise ValueError(f"No index named {index_name}")
         qr = ix_coll.find(where=where, limit=-1, **kwargs)
         index_col = ix.index_field
+        # TODO: optimize this for large indexes
         vector_pairs = [(row, np.array(row[index_col], dtype=float)) for row in qr.rows]
         results = ix.search(query, vector_pairs, limit=limit)
         for r in results:
@@ -305,11 +457,15 @@ class Collection:
 
         :return:
         """
-        if not self.name:
-            raise ValueError(f"Collection has no name: {self} // {self.metadata}")
-        return self.name.startswith("internal__")
+        if not self.alias:
+            raise ValueError(f"Collection has no alias: {self} // {self.metadata}")
+        return self.alias.startswith("internal__")
+
+    def load_from_source(self):
+        objects = load_objects(self.metadata.source_location)
+        self.insert(objects)
 
-    def attach_indexer(self, index: Union[Indexer, str], name: Optional[str] = True, auto_index=True, **kwargs):
+    def attach_indexer(self, index: Union[Indexer, str], name: Optional[str] = None, auto_index=True, **kwargs):
         """
         Attach an index to the collection.
 
@@ -333,6 +489,7 @@ class Collection:
         self._indexers[index_name] = index
         if auto_index:
             all_objs = self.find(limit=-1).rows
+            logger.info(f"Auto-indexing {len(all_objs)} objects")
             self.index_objects(all_objs, index_name, replace=True, **kwargs)
 
     def _index_collection_name(self, index_name: str) -> str:
@@ -340,6 +497,7 @@ class Collection:
         Create a name for a special collection that holds index data
 
         :param index_name:
+        :param indexer:
         :return:
         """
         return f"internal__index__{self.name}__{index_name}"
@@ -370,7 +528,9 @@ class Collection:
             logger.info(f"Checking if {ix_coll_name} is in {schema.classes.keys()}")
             if ix_coll_name in schema.classes:
                 ix_coll.delete_where()
+
         ix_coll.insert(objects_with_ix, **kwargs)
+        ix_coll.commit()
 
     def list_index_names(self) -> List[str]:
         """
@@ -405,12 +565,22 @@ class Collection:
 
         :return:
         """
-        sv = self.parent.schema_view
+        sv: SchemaView = self.parent.schema_view
         if sv:
             cls = sv.get_class(self.target_class_name)
+            if cls and not cls.attributes:
+                if not sv.class_induced_slots(cls.name):
+                    for att in self._induce_attributes():
+                        cls.attributes[att.name] = att
+                    sv.set_modified()
             return cls
         return None
 
+    def _induce_attributes(self) -> List[SlotDefinition]:
+        result = self.find({}, limit=-1)
+        cd = self.induce_class_definition_from_objects(result.rows, max_sample_size=None)
+        return list(cd.attributes.values())
+
     @property
     def identifier_attribute_name(self) -> Optional[str]:
         """
@@ -427,6 +597,37 @@ class Collection:
                     return att.name
         return None
 
+    def set_identifier_attribute_name(self, name: str):
+        """
+        Set the name of the identifier attribute for the collection.
+
+        AKA the primary key.
+
+        :param name: The name of the identifier attribute.
+        """
+        cd = self.class_definition()
+        if not cd:
+            raise ValueError(f"Cannot find class definition for {self.target_class_name}")
+        id_att = None
+        candidates = []
+        sv: SchemaView = self.parent.schema_view
+        cls = sv.get_class(cd.name)
+        existing_id_slot = sv.get_identifier_slot(cls.name)
+        if existing_id_slot:
+            if existing_id_slot.name == name:
+                return
+            existing_id_slot.identifier = False
+        for att in cls.attributes.values():
+            candidates.append(att.name)
+            if att.name == name:
+                att.identifier = True
+                id_att = att
+            else:
+                att.identifier = False
+        if not id_att:
+            raise ValueError(f"No attribute found with name {name} in {candidates}")
+        sv.set_modified()
+
     def object_identifier(self, obj: OBJECT, auto=True) -> Optional[IDENTIFIER]:
         """
         Return the identifier for an object.
@@ -457,6 +658,8 @@ class Collection:
         :param max_sample_size:
         :return:
         """
+        if not self.target_class_name:
+            raise ValueError(f"No target_class_name for {self.alias}")
         cd = ClassDefinition(self.target_class_name)
         keys = defaultdict(list)
         for obj in objs[0:max_sample_size]:
@@ -468,6 +671,8 @@ class Collection:
             for k, v in obj.items():
                 keys[k].append(v)
         for k, vs in keys.items():
+            if k == "_id":
+                continue
             multivalueds = []
             inlineds = []
             rngs = []
@@ -544,6 +749,39 @@ class Collection:
         """
         raise NotImplementedError
 
+    def apply_patches(self, patches: List[PatchDict], **kwargs):
+        """
+        Apply a patch to the collection.
+
+        Patches conform to the JSON Patch format,
+
+        :param patches:
+        :param kwargs:
+        :return:
+        """
+        all_objs = self.find(limit=-1).rows
+        primary_key = self.identifier_attribute_name
+        if not primary_key:
+            raise ValueError(f"No primary key for {self.target_class_name}")
+        new_objs = apply_patches_to_list(all_objs, patches, primary_key=primary_key, **kwargs)
+        self.replace(new_objs)
+
+    def diff(self, other: "Collection", **kwargs):
+        """
+        Diff two collections.
+
+        :param other:
+        :param kwargs:
+        :return:
+        """
+        src_objs = self.find(limit=-1).rows
+        tgt_objs = other.find(limit=-1).rows
+        primary_key = self.identifier_attribute_name
+        if not primary_key:
+            raise ValueError(f"No primary key for {self.target_class_name}")
+        patches_from_objects_lists(src_objs, tgt_objs, primary_key=primary_key)
+        return patches_from_objects_lists(src_objs, tgt_objs, primary_key=primary_key)
+
     def iter_validate_collection(self, **kwargs) -> Iterator["ValidationResult"]:
         """
         Validate the contents of the collection
@@ -563,3 +801,14 @@ class Collection:
         for obj in result.rows:
             obj = clean_empties(obj)
             yield from validator.iter_results(obj, class_name)
+
+    def commit(self):
+        """
+        Commit changes to the collection.
+
+        :return:
+        """
+        pass
+
+    def _broadcast(self, *args, **kwargs):
+        self.parent.broadcast(self, *args, **kwargs)

linkml_store/api/config.py

@@ -16,7 +16,7 @@ class CollectionConfig(BaseModel):
         default=None,
         description="The type of object in the collection. TODO; use this instead of name",
     )
-    metadata: Optional[Dict] = Field(
+    additional_properties: Optional[Dict] = Field(
         default=None,
         description="Optional metadata for the collection",
     )
@@ -36,6 +36,10 @@ class CollectionConfig(BaseModel):
         default=False,
         description="Whether the collection is prepopulated",
     )
+    source_location: Optional[str] = Field(
+        default=None,
+        description="Filesystem or remote URL that stores the data",
+    )
 
 
 class DatabaseConfig(BaseModel):
@@ -55,7 +59,7 @@ class DatabaseConfig(BaseModel):
         default=None,
         description="The LinkML schema as a dictionary",
     )
-    collections: Dict[str, CollectionConfig] = Field(
+    collections: Optional[Dict[str, CollectionConfig]] = Field(
         default={},
         description="A dictionary of collection configurations",
     )