linkml-store 0.1.6__tar.gz → 0.1.8__tar.gz

{linkml_store-0.1.6 → linkml_store-0.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: linkml-store
-Version: 0.1.6
+Version: 0.1.8
 Summary: linkml-store
 License: MIT
 Author: Author 1
@@ -27,6 +27,7 @@ Requires-Dist: click
 Requires-Dist: duckdb (>=0.10.1,<0.11.0)
 Requires-Dist: duckdb-engine (>=0.11.2)
 Requires-Dist: h5py ; extra == "h5py"
+Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: linkml ; extra == "validation"
 Requires-Dist: linkml-runtime (>=1.7.5,<2.0.0)
 Requires-Dist: linkml_map ; extra == "map"
@@ -53,3 +54,5 @@ There is also experimental support for vector-based indexing using OpenAI test e
 The goals of this project are to provide high level access to data stored in heterogeneous databases,
 with optional schema management using LinkML.
 
+See [these slides](https://docs.google.com/presentation/d/e/2PACX-1vSgtWUNUW0qNO_ZhMAGQ6fYhlXZJjBNMYT0OiZz8DDx8oj7iG9KofRs6SeaMXBBOICGknoyMG2zaHnm/embed?start=false&loop=false&delayms=3000) for more details
+

{linkml_store-0.1.6 → linkml_store-0.1.8}/README.md

@@ -8,3 +8,5 @@ There is also experimental support for vector-based indexing using OpenAI test e
 
 The goals of this project are to provide high level access to data stored in heterogeneous databases,
 with optional schema management using LinkML.
+
+See [these slides](https://docs.google.com/presentation/d/e/2PACX-1vSgtWUNUW0qNO_ZhMAGQ6fYhlXZJjBNMYT0OiZz8DDx8oj7iG9KofRs6SeaMXBBOICGknoyMG2zaHnm/embed?start=false&loop=false&delayms=3000) for more details

{linkml_store-0.1.6 → linkml_store-0.1.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "linkml-store"
-version = "0.1.6"
+version = "0.1.8"
 description = "linkml-store"
 authors = ["Author 1 <author@org.org>"]
 license = "MIT"
@@ -27,6 +27,7 @@ h5py = { version="*", optional = true }
 linkml = { version="*", optional = true }
 linkml_map = { version="*", optional = true }
 pandas = ">=2.2.1"
+jinja2 = "^3.1.4"
 
 [tool.poetry.group.dev.dependencies]
 pytest = {version = ">=7.1.2"}
@@ -36,10 +37,12 @@ sphinx = {version = ">=6.1.3"}
 sphinx-rtd-theme = {version = ">=1.0.0"}
 sphinx-autodoc-typehints = {version = "<2.0.0"}
 sphinx-click = {version = ">=4.3.0"}
+sphinx-automodapi = "*"
 myst-parser = {version = ">=0.18.1"}
 furo = {version = "*"}
 nbsphinx = "*"
 jupyter = "*"
+jupysql = "*"
 
 [tool.poetry.group.tests.dependencies]
 pytest = "^7.4.0"

{linkml_store-0.1.6 → linkml_store-0.1.8}/src/linkml_store/api/client.py

@@ -1,3 +1,4 @@
+import logging
 from pathlib import Path
 from typing import Dict, Optional, Union
 
@@ -11,6 +12,9 @@ from linkml_store.api.stores.duckdb.duckdb_database import DuckDBDatabase
 from linkml_store.api.stores.mongodb.mongodb_database import MongoDBDatabase
 from linkml_store.api.stores.solr.solr_database import SolrDatabase
 
+logger = logging.getLogger(__name__)
+
+
 HANDLE_MAP = {
     "duckdb": DuckDBDatabase,
     "solr": SolrDatabase,
@@ -21,12 +25,29 @@ HANDLE_MAP = {
 
 class Client:
     """
-    A client provides access to named collections.
+    A client is the top-level object for interacting with databases.
+
+    * A client has access to one or more :class:`.Database` objects.
+    * Each database consists of a number of :class:`.Collection` objects.
 
-    Examples
-    --------
+    Creating a client
+    -----------------
     >>> client = Client()
+
+    Attaching a database
+    --------------------
     >>> db = client.attach_database("duckdb", alias="test")
+
+    Note that normally a handle would be specified by a locator such as ``duckdb:///<PATH>``, but
+    for convenience, an in-memory duckdb object can be specified without a full locator
+
+    We can check the actual handle:
+
+    >>> db.handle
+    'duckdb:///:memory:'
+
+    Creating a new collection
+    -------------------------
     >>> collection = db.create_collection("Person")
     >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
     >>> collection.insert(objs)
@@ -147,6 +168,8 @@ class Client:
         if ":" not in handle:
             scheme = handle
             handle = None
+            if alias is None:
+                alias = scheme
         else:
             scheme, _ = handle.split(":", 1)
         if scheme not in HANDLE_MAP:
@@ -161,6 +184,11 @@ class Client:
             self._databases = {}
         self._databases[alias] = db
         db.parent = self
+        if db.alias:
+            if db.alias != alias:
+                raise AssertionError(f"Inconsistent alias: {db.alias} != {alias}")
+        else:
+            db.metadata.alias = alias
         return db
 
     def get_database(self, name: Optional[str] = None, create_if_not_exists=True, **kwargs) -> Database:
@@ -191,6 +219,7 @@ class Client:
             self._databases = {}
         if name not in self._databases:
             if create_if_not_exists:
+                logger.info(f"Creating database: {name}")
                 self.attach_database(name, **kwargs)
             else:
                 raise ValueError(f"Database {name} does not exist")

{linkml_store-0.1.6 → linkml_store-0.1.8}/src/linkml_store/api/collection.py

@@ -1,3 +1,5 @@
+"""A structure for representing collections of similar objects."""
+
 import hashlib
 import logging
 from collections import defaultdict
@@ -10,6 +12,8 @@ from linkml_runtime.linkml_model.meta import ArrayExpression
 from pydantic import BaseModel
 
 from linkml_store.index import get_indexer
+from linkml_store.utils.format_utils import load_objects
+from linkml_store.utils.object_utils import clean_empties
 
 try:
     from linkml.validator.report import ValidationResult
@@ -38,7 +42,17 @@ class Collection:
 
     - For relational databases, a collection is typically a table
     - For document databases such as MongoDB, a collection is the native type
-    - For a file system, a collection could be a single tabular file such as Parquet or CSV
+    - For a file system, a collection could be a single tabular file such as Parquet or CSV.
+
+    Collection objects are typically not created directly - instead they are generated
+    from a parent :class:`.Database` object:
+
+    >>> from linkml_store import Client
+    >>> client = Client()
+    >>> db = client.attach_database("duckdb", alias="test")
+    >>> collection = db.create_collection("Person")
+    >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+    >>> collection.insert(objs)
     """
 
     # name: str
@@ -56,25 +70,51 @@ class Collection:
             self.metadata = metadata
         else:
             self.metadata = CollectionConfig(name=name, **kwargs)
-        if name is not None and self.metadata.name is not None and name != self.metadata.name:
-            raise ValueError(f"Name mismatch: {name} != {self.metadata.name}")
+            if not self.metadata.alias:
+                self.metadata.alias = name
+            if not self.metadata.type:
+                self.metadata.type = name
+        # if name is not None and self.metadata.name is not None and name != self.metadata.name:
+        #    raise ValueError(f"Name mismatch: {name} != {self.metadata.name}")
 
     @property
     def name(self) -> str:
+        """
+        Return the name of the collection.
+
+        TODO: deprecate in favor of Type
+
+        :return: name of the collection
+        """
         return self.metadata.name
 
     @property
     def hidden(self) -> bool:
-        return self.metadata.hidden
+        """
+        True if the collection is hidden.
+
+        An example of a hidden collection is a collection that indexes another
+        collection
+
+        :return: True if the collection is hidden
+        """
+        # return self.metadata.hidden
 
     @property
-    def _target_class_name(self):
+    def target_class_name(self):
         """
         Return the name of the class that this collection represents
 
         This MUST be a LinkML class name
 
-        :return:
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person", alias="persons")
+        >>> collection.target_class_name
+        'Person'
+
+        :return: name of the class which members of this collection instantiate
         """
         # TODO: this is a shim layer until we can normalize on this
         if self.metadata.type:
@@ -82,7 +122,7 @@ class Collection:
         return self.name
 
     @property
-    def _alias(self):
+    def alias(self):
         """
         Return the primary name/alias used for the collection.
 
@@ -90,15 +130,34 @@ class Collection:
         to have an alias, for example "persons" which collects all instances
         of class Person.
 
-        The _alias SHOULD be used for Table names in SQL.
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person", alias="persons")
+        >>> collection.alias
+        'persons'
+
+        If no explicit alias is provided, then the target class name is used:
+
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person")
+        >>> collection.alias
+        'Person'
+
+        The alias SHOULD be used for Table names in SQL.
 
         For nested data, the alias SHOULD be used as the key; e.g
 
-         ``{ "persons": [ { "name": "Alice" }, { "name": "Bob" } ] }``
+        .. code-block:: json
+
+           { "persons": [ { "name": "Alice" }, { "name": "Bob" } ] }
 
         :return:
         """
         # TODO: this is a shim layer until we can normalize on this
+        # TODO: this is a shim layer until we can normalize on this
         if self.metadata.alias:
             return self.metadata.alias
         return self.name
@@ -107,6 +166,13 @@ class Collection:
         """
         Replace entire collection with objects.
 
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person")
+        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+        >>> collection.insert(objs)
+
         :param objs:
         :param kwargs:
         :return:
@@ -116,7 +182,14 @@ class Collection:
 
     def insert(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         """
-        Add one or more objects to the collection
+        Add one or more objects to the collection.
+
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person")
+        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+        >>> collection.insert(objs)
 
         :param objs:
         :param kwargs:
@@ -124,9 +197,32 @@ class Collection:
         """
         raise NotImplementedError
 
-    def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> int:
+    def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> Optional[int]:
         """
-        Delete one or more objects from the collection
+        Delete one or more objects from the collection.
+
+        First let's set up a collection:
+
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person")
+        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+        >>> collection.insert(objs)
+        >>> collection.find({}).num_rows
+        2
+
+        Now let's delete an object:
+
+        >>> collection.delete(objs[0])
+        >>> collection.find({}).num_rows
+        1
+
+        Deleting the same object again should have no effect:
+
+        >>> collection.delete(objs[0])
+        >>> collection.find({}).num_rows
+        1
 
         :param objs:
         :param kwargs:
@@ -134,9 +230,30 @@ class Collection:
         """
         raise NotImplementedError
 
-    def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> int:
+    def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> Optional[int]:
         """
-        Delete objects that match a query
+        Delete objects that match a query.
+
+        First let's set up a collection:
+
+        >>> from linkml_store import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> collection = db.create_collection("Person")
+        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+        >>> collection.insert(objs)
+
+        Now let's delete an object:
+
+        >>> collection.delete_where({"id": "P1"})
+        >>> collection.find({}).num_rows
+        1
+
+        Match everything:
+
+        >>> collection.delete_where({})
+        >>> collection.find({}).num_rows
+        0
 
         :param where: where conditions
         :param missing_ok: if True, do not raise an error if the collection does not exist
@@ -147,7 +264,7 @@ class Collection:
 
     def update(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         """
-        Update one or more objects in the collection
+        Update one or more objects in the collection.
 
         :param objs:
         :param kwargs:
@@ -156,11 +273,25 @@ class Collection:
         raise NotImplementedError
 
     def _create_query(self, **kwargs) -> Query:
-        return Query(from_table=self._alias, **kwargs)
+        return Query(from_table=self.alias, **kwargs)
 
     def query(self, query: Query, **kwargs) -> QueryResult:
         """
-        Run a query against the collection
+        Run a query against the collection.
+
+        First let's load a collection:
+
+        >>> from linkml_store import Client
+        >>> from linkml_store.utils.format_utils import load_objects
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb")
+        >>> collection = db.create_collection("Country")
+        >>> objs = load_objects("tests/input/countries/countries.jsonl")
+        >>> collection.insert(objs)
+
+        Now let's run a query:
+
+        TODO
 
         :param query:
         :param kwargs:
@@ -193,7 +324,7 @@ class Collection:
         """
         raise NotImplementedError
 
-    def get(self, ids: Optional[IDENTIFIER], **kwargs) -> QueryResult:
+    def get(self, ids: Optional[List[IDENTIFIER]], **kwargs) -> QueryResult:
         """
         Get one or more objects by ID.
 
@@ -201,14 +332,60 @@ class Collection:
         :param kwargs:
         :return:
         """
-        id_field = self.identifier_field
-        q = self._create_query(where_clause={id_field: ids})
-        return self.query(q, **kwargs)
+        # TODO
+        id_field = self.identifier_attribute_name
+        if not id_field:
+            raise ValueError(f"No identifier for {self.name}")
+        return self.find({id_field: ids})
+
+    def get_one(self, id: IDENTIFIER, **kwargs) -> Optional[OBJECT]:
+        """
+        Get one object by ID.
+
+        :param id:
+        :param kwargs:
+        :return:
+        """
+        if not id:
+            raise ValueError("Must pass an ID")
+        id_field = self.identifier_attribute_name
+        if not id_field:
+            raise ValueError(f"No identifier for {self.name}")
+        w = {id_field: id}
+        qr = self.find(w)
+        if qr.num_rows == 1:
+            return qr.rows[0]
+        return None
 
     def find(self, where: Optional[Any] = None, **kwargs) -> QueryResult:
         """
         Find objects in the collection using a where query.
 
+        As an example, first load a collection:
+
+        >>> from linkml_store import Client
+        >>> from linkml_store.utils.format_utils import load_objects
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb")
+        >>> collection = db.create_collection("Country")
+        >>> objs = load_objects("tests/input/countries/countries.jsonl")
+        >>> collection.insert(objs)
+
+        Now let's find all objects:
+
+        >>> qr = collection.find({})
+        >>> qr.num_rows
+        20
+
+        We can do a more restrictive query:
+
+        >>> qr = collection.find({"code": "FR"})
+        >>> qr.num_rows
+        1
+        >>> qr.rows[0]["name"]
+        'France'
+
+
         :param where:
         :param kwargs:
         :return:
@@ -216,6 +393,18 @@ class Collection:
         query = self._create_query(where_clause=where)
         return self.query(query, **kwargs)
 
+    def find_iter(self, where: Optional[Any] = None, **kwargs) -> Iterator[OBJECT]:
+        """
+        Find objects in the collection using a where query.
+
+        :param where:
+        :param kwargs:
+        :return:
+        """
+        qr = self.find(where=where, limit=-1, **kwargs)
+        for row in qr.rows:
+            yield row
+
     def search(
         self,
         query: str,
@@ -245,6 +434,7 @@ class Collection:
             raise ValueError(f"No index named {index_name}")
         qr = ix_coll.find(where=where, limit=-1, **kwargs)
         index_col = ix.index_field
+        # TODO: optimize this for large indexes
         vector_pairs = [(row, np.array(row[index_col], dtype=float)) for row in qr.rows]
         results = ix.search(query, vector_pairs, limit=limit)
         for r in results:
@@ -260,11 +450,15 @@ class Collection:
 
         :return:
         """
-        if not self.name:
-            raise ValueError(f"Collection has no name: {self} // {self.metadata}")
-        return self.name.startswith("internal__")
+        if not self.alias:
+            raise ValueError(f"Collection has no alias: {self} // {self.metadata}")
+        return self.alias.startswith("internal__")
 
-    def attach_indexer(self, index: Union[Indexer, str], name: Optional[str] = True, auto_index=True, **kwargs):
+    def load_from_source(self):
+        objects = load_objects(self.metadata.source_location)
+        self.insert(objects)
+
+    def attach_indexer(self, index: Union[Indexer, str], name: Optional[str] = None, auto_index=True, **kwargs):
         """
         Attach an index to the collection.
 
@@ -288,6 +482,7 @@ class Collection:
         self._indexers[index_name] = index
         if auto_index:
             all_objs = self.find(limit=-1).rows
+            logger.info(f"Auto-indexing {len(all_objs)} objects")
             self.index_objects(all_objs, index_name, replace=True, **kwargs)
 
     def _index_collection_name(self, index_name: str) -> str:
@@ -295,6 +490,7 @@ class Collection:
         Create a name for a special collection that holds index data
 
         :param index_name:
+        :param indexer:
         :return:
         """
         return f"internal__index__{self.name}__{index_name}"
@@ -325,6 +521,7 @@ class Collection:
             logger.info(f"Checking if {ix_coll_name} is in {schema.classes.keys()}")
             if ix_coll_name in schema.classes:
                 ix_coll.delete_where()
+
         ix_coll.insert(objects_with_ix, **kwargs)
 
     def list_index_names(self) -> List[str]:
@@ -362,10 +559,11 @@ class Collection:
         """
         sv = self.parent.schema_view
         if sv:
-            cls = sv.get_class(self._target_class_name)
+            cls = sv.get_class(self.target_class_name)
             return cls
         return None
 
+    @property
     def identifier_attribute_name(self) -> Optional[str]:
         """
         Return the name of the identifier attribute for the collection.
@@ -376,7 +574,7 @@ class Collection:
         """
         cd = self.class_definition()
         if cd:
-            for att in cd.attributes.values():
+            for att in self.parent.schema_view.class_induced_slots(cd.name):
                 if att.identifier:
                     return att.name
         return None
@@ -411,7 +609,9 @@ class Collection:
         :param max_sample_size:
         :return:
         """
-        cd = ClassDefinition(self._target_class_name)
+        if not self.target_class_name:
+            raise ValueError(f"No target_class_name for {self.alias}")
+        cd = ClassDefinition(self.target_class_name)
         keys = defaultdict(list)
         for obj in objs[0:max_sample_size]:
             if isinstance(obj, BaseModel):
@@ -474,7 +674,7 @@ class Collection:
                 array_expr = ArrayExpression(exact_number_dimensions=len(exact_dimensions_list[0]))
                 cd.attributes[k].array = array_expr
         sv = self.parent.schema_view
-        sv.schema.classes[self._target_class_name] = cd
+        sv.schema.classes[self.target_class_name] = cd
         sv.set_modified()
         return cd
 
@@ -511,8 +711,9 @@ class Collection:
         validator = Validator(self.parent.schema_view.schema, validation_plugins=validation_plugins)
         cd = self.class_definition()
         if not cd:
-            raise ValueError(f"Cannot find class definition for {self._target_class_name}")
+            raise ValueError(f"Cannot find class definition for {self.target_class_name}")
         class_name = cd.name
         result = self.find(**kwargs)
         for obj in result.rows:
+            obj = clean_empties(obj)
             yield from validator.iter_results(obj, class_name)

{linkml_store-0.1.6 → linkml_store-0.1.8}/src/linkml_store/api/config.py

@@ -16,7 +16,7 @@ class CollectionConfig(BaseModel):
         default=None,
         description="The type of object in the collection. TODO; use this instead of name",
     )
-    metadata: Optional[Dict] = Field(
+    additional_properties: Optional[Dict] = Field(
         default=None,
         description="Optional metadata for the collection",
     )
@@ -36,6 +36,10 @@ class CollectionConfig(BaseModel):
         default=False,
         description="Whether the collection is prepopulated",
     )
+    source_location: Optional[str] = Field(
+        default=None,
+        description="Filesystem or remote URL that stores the data",
+    )
 
 
 class DatabaseConfig(BaseModel):
@@ -55,7 +59,7 @@ class DatabaseConfig(BaseModel):
         default=None,
         description="The LinkML schema as a dictionary",
     )
-    collections: Dict[str, CollectionConfig] = Field(
+    collections: Optional[Dict[str, CollectionConfig]] = Field(
         default={},
         description="A dictionary of collection configurations",
     )
@@ -76,6 +80,10 @@ class DatabaseConfig(BaseModel):
         default=None,
         description="Optional configuration for search fields",
     )
+    ensure_referential_integrity: bool = Field(
+        default=False,
+        description="Whether to ensure referential integrity",
+    )
 
 
 class ClientConfig(BaseModel):