linkml-store 0.1.6__tar.gz → 0.1.7__tar.gz

{linkml_store-0.1.6 → linkml_store-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: linkml-store
-Version: 0.1.6
+Version: 0.1.7
 Summary: linkml-store
 License: MIT
 Author: Author 1

{linkml_store-0.1.6 → linkml_store-0.1.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "linkml-store"
-version = "0.1.6"
+version = "0.1.7"
 description = "linkml-store"
 authors = ["Author 1 <author@org.org>"]
 license = "MIT"
@@ -36,6 +36,7 @@ sphinx = {version = ">=6.1.3"}
 sphinx-rtd-theme = {version = ">=1.0.0"}
 sphinx-autodoc-typehints = {version = "<2.0.0"}
 sphinx-click = {version = ">=4.3.0"}
+sphinx-automodapi = "*"
 myst-parser = {version = ">=0.18.1"}
 furo = {version = "*"}
 nbsphinx = "*"

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/client.py

@@ -21,7 +21,11 @@ HANDLE_MAP = {
 
 class Client:
     """
-    A client provides access to named collections.
+    A client is the top-level object for interacting with databases.
+
+    A client has access to one or more :class:`Database` objects.
+
+    Each database consists of a number of :class:`.Collection` objects.
 
     Examples
     --------

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/collection.py

@@ -10,6 +10,7 @@ from linkml_runtime.linkml_model.meta import ArrayExpression
 from pydantic import BaseModel
 
 from linkml_store.index import get_indexer
+from linkml_store.utils.object_utils import clean_empties
 
 try:
     from linkml.validator.report import ValidationResult
@@ -61,14 +62,27 @@ class Collection:
 
     @property
     def name(self) -> str:
+        """
+        Return the name of the collection
+
+        :return:
+        """
         return self.metadata.name
 
     @property
     def hidden(self) -> bool:
+        """
+        True if the collection is hidden.
+
+        An example of a hidden collection is a collection that indexes another
+        collection
+
+        :return: True if the collection is hidden
+        """
         return self.metadata.hidden
 
     @property
-    def _target_class_name(self):
+    def target_class_name(self):
         """
         Return the name of the class that this collection represents
 
@@ -82,7 +96,7 @@ class Collection:
         return self.name
 
     @property
-    def _alias(self):
+    def alias(self):
         """
         Return the primary name/alias used for the collection.
 
@@ -156,7 +170,7 @@ class Collection:
         raise NotImplementedError
 
     def _create_query(self, **kwargs) -> Query:
-        return Query(from_table=self._alias, **kwargs)
+        return Query(from_table=self.alias, **kwargs)
 
     def query(self, query: Query, **kwargs) -> QueryResult:
         """
@@ -201,9 +215,28 @@ class Collection:
         :param kwargs:
         :return:
         """
-        id_field = self.identifier_field
-        q = self._create_query(where_clause={id_field: ids})
-        return self.query(q, **kwargs)
+        # TODO
+        id_field = self.identifier_attribute_name
+        return self.find({id_field: ids})
+
+    def get_one(self, id: IDENTIFIER, **kwargs) -> Optional[OBJECT]:
+        """
+        Get one object by ID.
+
+        :param id:
+        :param kwargs:
+        :return:
+        """
+        if not id:
+            raise ValueError("Must pass an ID")
+        id_field = self.identifier_attribute_name
+        if not id_field:
+            raise ValueError(f"No identifier for {self.name}")
+        w = {id_field: id}
+        qr = self.find(w)
+        if qr.num_rows == 1:
+            return qr.rows[0]
+        return None
 
     def find(self, where: Optional[Any] = None, **kwargs) -> QueryResult:
         """
@@ -216,6 +249,18 @@ class Collection:
         query = self._create_query(where_clause=where)
         return self.query(query, **kwargs)
 
+    def find_iter(self, where: Optional[Any] = None, **kwargs) -> Iterator[OBJECT]:
+        """
+        Find objects in the collection using a where query.
+
+        :param where:
+        :param kwargs:
+        :return:
+        """
+        qr = self.find(where=where, limit=-1, **kwargs)
+        for row in qr.rows:
+            yield row
+
     def search(
         self,
         query: str,
@@ -362,10 +407,11 @@ class Collection:
         """
         sv = self.parent.schema_view
         if sv:
-            cls = sv.get_class(self._target_class_name)
+            cls = sv.get_class(self.target_class_name)
             return cls
         return None
 
+    @property
     def identifier_attribute_name(self) -> Optional[str]:
         """
         Return the name of the identifier attribute for the collection.
@@ -376,7 +422,7 @@ class Collection:
         """
         cd = self.class_definition()
         if cd:
-            for att in cd.attributes.values():
+            for att in self.parent.schema_view.class_induced_slots(cd.name):
                 if att.identifier:
                     return att.name
         return None
@@ -411,7 +457,7 @@ class Collection:
         :param max_sample_size:
         :return:
         """
-        cd = ClassDefinition(self._target_class_name)
+        cd = ClassDefinition(self.target_class_name)
         keys = defaultdict(list)
         for obj in objs[0:max_sample_size]:
             if isinstance(obj, BaseModel):
@@ -474,7 +520,7 @@ class Collection:
                 array_expr = ArrayExpression(exact_number_dimensions=len(exact_dimensions_list[0]))
                 cd.attributes[k].array = array_expr
         sv = self.parent.schema_view
-        sv.schema.classes[self._target_class_name] = cd
+        sv.schema.classes[self.target_class_name] = cd
         sv.set_modified()
         return cd
 
@@ -511,8 +557,9 @@ class Collection:
         validator = Validator(self.parent.schema_view.schema, validation_plugins=validation_plugins)
         cd = self.class_definition()
         if not cd:
-            raise ValueError(f"Cannot find class definition for {self._target_class_name}")
+            raise ValueError(f"Cannot find class definition for {self.target_class_name}")
         class_name = cd.name
         result = self.find(**kwargs)
         for obj in result.rows:
+            obj = clean_empties(obj)
             yield from validator.iter_results(obj, class_name)

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/config.py

@@ -76,6 +76,10 @@ class DatabaseConfig(BaseModel):
         default=None,
         description="Optional configuration for search fields",
     )
+    ensure_referential_integrity: bool = Field(
+        default=False,
+        description="Whether to ensure referential integrity",
+    )
 
 
 class ClientConfig(BaseModel):

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/database.py

@@ -1,11 +1,12 @@
 import logging
 from abc import ABC
+from collections import defaultdict
 from copy import copy
 from pathlib import Path
 from typing import TYPE_CHECKING, ClassVar, Dict, Iterator, Optional, Sequence, Type, Union
 
 try:
-    from linkml.validator.report import ValidationResult
+    from linkml.validator.report import Severity, ValidationResult
 except ImportError:
     ValidationResult = None
 
@@ -204,7 +205,6 @@ class Database(ABC):
         """
         if not name:
             raise ValueError(f"Collection name must be provided: alias: {alias} metadata: {metadata}")
-        # collection_cls = self._collection_class
         collection_cls = self.collection_class
         collection = collection_cls(name=name, alias=alias, parent=self, metadata=metadata)
         if metadata and metadata.attributes:
@@ -341,14 +341,42 @@ class Database(ABC):
             self._schema_view = self.induce_schema_view()
         return self._schema_view
 
-    def set_schema_view(self, schema_view: SchemaView):
+    def set_schema_view(self, schema_view: Union[str, Path, SchemaView]):
         """
         Set the schema view for the database.
 
         :param schema_view:
         :return:
         """
+        if isinstance(schema_view, Path):
+            schema_view = str(schema_view)
+        if isinstance(schema_view, str):
+            schema_view = SchemaView(schema_view)
         self._schema_view = schema_view
+        if not self._collections:
+            return
+        # align with induced schema
+        roots = [c for c in schema_view.all_classes().values() if c.tree_root]
+        if len(roots) == 0:
+            all_ranges = set()
+            for cn in schema_view.all_classes():
+                for slot in schema_view.class_induced_slots(cn):
+                    if slot.range:
+                        all_ranges.add(slot.range)
+            roots = [
+                c
+                for c in schema_view.all_classes().values()
+                if not all_ranges.intersection(schema_view.class_ancestors(c.name, reflexive=True))
+            ]
+        if len(roots) == 1:
+            root = roots[0]
+            for slot in schema_view.class_induced_slots(root.name):
+                inlined = slot.inlined or slot.inlined_as_list
+                if inlined and slot.range:
+                    if slot.name in self._collections:
+                        coll = self._collections[slot.name]
+                        if not coll.metadata.type:
+                            coll.metadata.type = slot.range
 
     def load_schema_view(self, path: Union[str, Path]):
         """
@@ -397,6 +425,52 @@ class Database(ABC):
         """
         for collection in self.list_collections():
             yield from collection.iter_validate_collection(**kwargs)
+        if self.metadata.ensure_referential_integrity:
+            yield from self._validate_referential_integrity(**kwargs)
+
+    def _validate_referential_integrity(self, **kwargs) -> Iterator["ValidationResult"]:
+        """
+        Validate referential integrity of the database.
+
+        :param kwargs:
+        :return: iterator over validation results
+        """
+        sv = self.schema_view
+        cmap = defaultdict(list)
+        for collection in self.list_collections():
+            if not collection.target_class_name:
+                raise ValueError(f"Collection {collection.name} has no target class")
+            cmap[collection.target_class_name].append(collection)
+        for collection in self.list_collections():
+            cd = collection.class_definition()
+            induced_slots = sv.class_induced_slots(cd.name)
+            slot_map = {s.name: s for s in induced_slots}
+            # rmap = {s.name: s.range for s in induced_slots}
+            sr_to_coll = {s.name: cmap.get(s.range, []) for s in induced_slots if s.range}
+            for obj in collection.find_iter():
+                for k, v in obj.items():
+                    if k not in sr_to_coll:
+                        continue
+                    ref_colls = sr_to_coll[k]
+                    if not ref_colls:
+                        continue
+                    if not isinstance(v, (str, int)):
+                        continue
+                    slot = slot_map[k]
+                    found = False
+                    for ref_coll in ref_colls:
+                        ref_obj = ref_coll.get_one(v)
+                        if ref_obj:
+                            found = True
+                            break
+                    if not found:
+                        yield ValidationResult(
+                            type="ReferentialIntegrity",
+                            severity=Severity.ERROR,
+                            message=f"Referential integrity error: {slot.range} not found",
+                            instantiates=slot.range,
+                            instance=v,
+                        )
 
     def drop(self, **kwargs):
         """

linkml_store-0.1.7/src/linkml_store/api/stores/chromadb/__init__.py

@@ -0,0 +1,3 @@
+"""
+Support for ChromaDB is experimental.
+"""

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/stores/chromadb/chromadb_collection.py

@@ -1,3 +1,7 @@
+"""
+ChromaDB Collection
+"""
+
 import logging
 from typing import Any, Dict, List, Optional, Tuple, Union
 
@@ -13,6 +17,9 @@ logger = logging.getLogger(__name__)
 
 
 class ChromaDBCollection(Collection):
+    """
+    A wrapper for ChromaDB collections.
+    """
 
     @property
     def native_collection(self) -> ChromaCollection:
@@ -50,7 +57,7 @@ class ChromaDBCollection(Collection):
         return len(ids)
 
     def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> int:
-        logger.info(f"Deleting from {self._target_class_name} where: {where}")
+        logger.info(f"Deleting from {self.target_class_name} where: {where}")
         if where is None:
             where = {}
         results = self.native_collection.get(where=where)

linkml_store-0.1.7/src/linkml_store/api/stores/duckdb/__init__.py

@@ -0,0 +1,7 @@
+from linkml_store.api.stores.duckdb.duckdb_collection import DuckDBCollection
+from linkml_store.api.stores.duckdb.duckdb_database import DuckDBDatabase
+
+__all__ = [
+    "DuckDBCollection",
+    "DuckDBDatabase",
+]

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/stores/duckdb/duckdb_collection.py

@@ -28,7 +28,7 @@ class DuckDBCollection(Collection):
             cd = self.induce_class_definition_from_objects(objs)
         self._create_table(cd)
         table = self._sqla_table(cd)
-        logger.info(f"Inserting into: {self._alias} // T={table.name}")
+        logger.info(f"Inserting into: {self.alias} // T={table.name}")
         engine = self.parent.engine
         col_names = [c.name for c in table.columns]
         objs = [{k: obj.get(k, None) for k in col_names} for obj in objs]
@@ -55,12 +55,12 @@ class DuckDBCollection(Collection):
         return len(objs)
 
     def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> int:
-        logger.info(f"Deleting from {self._target_class_name} where: {where}")
+        logger.info(f"Deleting from {self.target_class_name} where: {where}")
         if where is None:
             where = {}
         cd = self.class_definition()
         if not cd:
-            logger.info(f"No class definition found for {self._target_class_name}, assuming not prepopulated")
+            logger.info(f"No class definition found for {self.target_class_name}, assuming not prepopulated")
             return 0
         table = self._sqla_table(cd)
         engine = self.parent.engine
@@ -115,7 +115,7 @@ class DuckDBCollection(Collection):
                 typ = sqla.ARRAY(typ, dimensions=1)
             col = Column(att.name, typ)
             cols.append(col)
-        t = Table(self._alias, metadata_obj, *cols)
+        t = Table(self.alias, metadata_obj, *cols)
         return t
 
     def _create_table(self, cd: ClassDefinition):
@@ -123,7 +123,7 @@ class DuckDBCollection(Collection):
             logger.info(f"Already have table for: {cd.name}")
             return
         query = Query(
-            from_table="information_schema.tables", where_clause={"table_type": "BASE TABLE", "table_name": self._alias}
+            from_table="information_schema.tables", where_clause={"table_type": "BASE TABLE", "table_name": self.alias}
         )
         qr = self.parent.query(query)
         if qr.num_rows > 0:

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/stores/duckdb/duckdb_database.py

@@ -73,12 +73,12 @@ class DuckDBDatabase(Database):
             if sv:
                 cd = None
                 for c in self._collections.values():
-                    if c.name == query.from_table:
+                    if c.name == query.from_table or c.metadata.alias == query.from_table:
                         cd = c.class_definition()
                         break
                 if cd:
-                    for att in cd.attributes.values():
-                        if att.inlined:
+                    for att in sv.class_induced_slots(cd.name):
+                        if att.inlined or att.inlined_as_list:
                             json_encoded_cols.append(att.name)
         with self.engine.connect() as conn:
             count_query_str = text(query_to_sql(query, count=True))

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/stores/hdf5/hdf5_collection.py

@@ -46,7 +46,7 @@ class HDF5Collection(Collection):
         return count
 
     def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> int:
-        logger.info(f"Deleting from {self._target_class_name} where: {where}")
+        logger.info(f"Deleting from {self.target_class_name} where: {where}")
         if where is None:
             where = {}
         results = self.query(Query(where_clause=where)).rows

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/stores/mongodb/mongodb_collection.py

@@ -1,4 +1,5 @@
 import logging
+from copy import copy
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 from linkml_runtime.linkml_model import SlotDefinition
@@ -31,7 +32,12 @@ class MongoDBCollection(Collection):
         else:
             cursor = self.mongo_collection.find(mongo_filter)
 
-        rows = list(cursor)
+        def _as_row(row: dict):
+            row = copy(row)
+            del row["_id"]
+            return row
+
+        rows = [_as_row(row) for row in cursor]
         count = self.mongo_collection.count_documents(mongo_filter)
 
         return QueryResult(query=query, num_rows=count, rows=rows)
@@ -92,7 +98,7 @@ class MongoDBCollection(Collection):
         return result.deleted_count
 
     def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> int:
-        logger.info(f"Deleting from {self._target_class_name} where: {where}")
+        logger.info(f"Deleting from {self.target_class_name} where: {where}")
         if where is None:
             where = {}
         result = self.mongo_collection.delete_many(where)

linkml_store-0.1.7/src/linkml_store/api/stores/solr/__init__.py

@@ -0,0 +1,3 @@
+"""
+Wrapper for Solr endpoints.
+"""

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/stores/solr/solr_collection.py

@@ -5,6 +5,7 @@ from copy import copy
 from typing import Any, Dict, List, Optional, Union
 
 import requests
+
 from linkml_store.api import Collection
 from linkml_store.api.collection import DEFAULT_FACET_LIMIT
 from linkml_store.api.queries import Query, QueryResult
@@ -119,7 +120,7 @@ class SolrCollection(Collection):
         conditions = []
         if self.parent.metadata.collection_type_slot:
             where_clause = copy(where_clause)
-            where_clause[self.parent.metadata.collection_type_slot] = self._alias
+            where_clause[self.parent.metadata.collection_type_slot] = self.alias
         for field, value in where_clause.items():
             if not isinstance(value, (list, tuple)):
                 value = [value]

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/api/stores/solr/solr_database.py

@@ -2,6 +2,7 @@ import logging
 from typing import Optional
 
 import requests
+
 from linkml_store.api import Collection, Database
 from linkml_store.api.config import CollectionConfig
 from linkml_store.api.queries import Query, QueryResult

{linkml_store-0.1.6 → linkml_store-0.1.7}/src/linkml_store/utils/object_utils.py

@@ -1,6 +1,6 @@
 import json
 from copy import deepcopy
-from typing import Any, Dict, Union
+from typing import Any, Dict, List, Union
 
 from pydantic import BaseModel
 
@@ -71,3 +71,11 @@ def parse_update_expression(expr: str) -> Union[tuple[str, Any], None]:
     except ValueError:
         return None
     return path, val
+
+
+def clean_empties(value: Union[Dict, List]) -> Any:
+    if isinstance(value, dict):
+        value = {k: v for k, v in ((k, clean_empties(v)) for k, v in value.items()) if v is not None}
+    elif isinstance(value, list):
+        value = [v for v in (clean_empties(v) for v in value) if v is not None]
+    return value