linkml-store 0.1.8__py3-none-any.whl → 0.1.10__py3-none-any.whl

linkml_store/api/client.py

@@ -9,6 +9,7 @@ from linkml_store.api import Database
 from linkml_store.api.config import ClientConfig
 from linkml_store.api.stores.chromadb.chromadb_database import ChromaDBDatabase
 from linkml_store.api.stores.duckdb.duckdb_database import DuckDBDatabase
+from linkml_store.api.stores.filesystem.filesystem_database import FileSystemDatabase
 from linkml_store.api.stores.mongodb.mongodb_database import MongoDBDatabase
 from linkml_store.api.stores.solr.solr_database import SolrDatabase
 
@@ -20,6 +21,7 @@ HANDLE_MAP = {
     "solr": SolrDatabase,
     "mongodb": MongoDBDatabase,
     "chromadb": ChromaDBDatabase,
+    "file": FileSystemDatabase,
 }
 
 
@@ -96,7 +98,7 @@ class Client:
         """
         return self.metadata.base_dir
 
-    def from_config(self, config: Union[ClientConfig, str, Path], base_dir=None, **kwargs):
+    def from_config(self, config: Union[ClientConfig, dict, str, Path], base_dir=None, **kwargs):
         """
         Create a client from a configuration.
 
@@ -116,11 +118,13 @@ class Client:
         :return:
 
         """
+        if isinstance(config, dict):
+            config = ClientConfig(**config)
         if isinstance(config, Path):
             config = str(config)
         if isinstance(config, str):
-            if not base_dir:
-                base_dir = Path(config).parent
+            # if not base_dir:
+            #    base_dir = Path(config).parent
             parsed_obj = yaml.safe_load(open(config))
             config = ClientConfig(**parsed_obj)
         self.metadata = config
@@ -131,8 +135,15 @@ class Client:
 
     def _initialize_databases(self, **kwargs):
         for name, db_config in self.metadata.databases.items():
-            handle = db_config.handle.format(base_dir=self.base_dir)
+            base_dir = self.base_dir
+            logger.info(f"Initializing database: {name}, base_dir: {base_dir}")
+            if not base_dir:
+                base_dir = Path.cwd()
+                logger.info(f"Using current working directory: {base_dir}")
+            handle = db_config.handle.format(base_dir=base_dir)
             db_config.handle = handle
+            if db_config.schema_location:
+                db_config.schema_location = db_config.schema_location.format(base_dir=base_dir)
             db = self.attach_database(handle, alias=name, **kwargs)
             db.from_config(db_config)
 

linkml_store/api/collection.py

@@ -4,16 +4,19 @@ import hashlib
 import logging
 from collections import defaultdict
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, TextIO, Type, Union
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, Generic, Iterator, List, Optional, TextIO, Tuple, Type, Union
 
 import numpy as np
+from linkml_runtime import SchemaView
 from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition
 from linkml_runtime.linkml_model.meta import ArrayExpression
 from pydantic import BaseModel
 
+from linkml_store.api.types import DatabaseType
 from linkml_store.index import get_indexer
 from linkml_store.utils.format_utils import load_objects
 from linkml_store.utils.object_utils import clean_empties
+from linkml_store.utils.patch_utils import PatchDict, apply_patches_to_list, patches_from_objects_lists
 
 try:
     from linkml.validator.report import ValidationResult
@@ -36,7 +39,7 @@ IDENTIFIER = str
 FIELD_NAME = str
 
 
-class Collection:
+class Collection(Generic[DatabaseType]):
     """
     A collection is an organized set of objects of the same or similar type.
 
@@ -56,11 +59,12 @@ class Collection:
     """
 
     # name: str
-    parent: Optional["Database"] = None
+    parent: Optional[DatabaseType] = None
     _indexers: Optional[Dict[str, Indexer]] = None
     # hidden: Optional[bool] = False
 
     metadata: Optional[CollectionConfig] = None
+    default_index_name: ClassVar[str] = "simple"
 
     def __init__(
         self, name: str, parent: Optional["Database"] = None, metadata: Optional[CollectionConfig] = None, **kwargs
@@ -197,6 +201,10 @@ class Collection:
         """
         raise NotImplementedError
 
+    def _post_insert_hook(self, objs: List[OBJECT], **kwargs):
+        patches = [{"op": "add", "path": "/0", "value": obj} for obj in objs]
+        self._broadcast(patches, **kwargs)
+
     def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> Optional[int]:
         """
         Delete one or more objects from the collection.
@@ -301,7 +309,7 @@ class Collection:
 
     def query_facets(
         self, where: Optional[Dict] = None, facet_columns: List[str] = None, facet_limit=DEFAULT_FACET_LIMIT, **kwargs
-    ) -> Dict[str, Dict[str, int]]:
+    ) -> Dict[str, List[Tuple[Any, int]]]:
         """
         Run a query to get facet counts for one or more columns.
 
@@ -319,7 +327,7 @@ class Collection:
         :param query: A Query object representing the base query.
         :param facet_columns: A list of column names to get facet counts for.
         :param facet_limit:
-        :return: A dictionary where keys are column names and values are pandas DataFrames
+        :return: A dictionary where keys are column names and values are tuples
                  containing the facet counts for each unique value in the respective column.
         """
         raise NotImplementedError
@@ -414,7 +422,30 @@ class Collection:
         **kwargs,
     ) -> QueryResult:
         """
-        Search the collection using a full-text search index.
+        Search the collection using a text-based index index.
+
+        Example:
+
+        >>> from linkml_store import Client
+        >>> from linkml_store.utils.format_utils import load_objects
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb")
+        >>> collection = db.create_collection("Country")
+        >>> objs = load_objects("tests/input/countries/countries.jsonl")
+        >>> collection.insert(objs)
+
+        Now let's index, using the simple trigram-based index
+
+        >>> index = get_indexer("simple")
+        >>> collection.attach_indexer(index)
+
+        Now let's find all objects:
+
+        >>> qr = collection.search("France")
+        >>> score, top_obj = qr.ranked_rows[0]
+        >>> assert score > 0.1
+        >>> top_obj["code"]
+        'FR'
 
         :param query:
         :param where:
@@ -424,12 +455,18 @@ class Collection:
         :return:
         """
         if index_name is None:
-            if len(self._indexers) == 1:
-                index_name = list(self._indexers.keys())[0]
+            if len(self.indexers) == 1:
+                index_name = list(self.indexers.keys())[0]
             else:
-                raise ValueError("Multiple indexes found. Please specify an index name.")
+                logger.warning("Multiple indexes found. Using default index.")
+                index_name = self.default_index_name
         ix_coll = self.parent.get_collection(self._index_collection_name(index_name))
-        ix = self._indexers.get(index_name)
+        if index_name not in self.indexers:
+            ix = get_indexer(index_name)
+            if not self._indexers:
+                self._indexers = {}
+            self._indexers[index_name] = ix
+        ix = self.indexers.get(index_name)
         if not ix:
             raise ValueError(f"No index named {index_name}")
         qr = ix_coll.find(where=where, limit=-1, **kwargs)
@@ -446,7 +483,10 @@ class Collection:
     @property
     def is_internal(self) -> bool:
         """
-        Check if the collection is internal
+        Check if the collection is internal.
+
+        Internal collections are hidden by default. Examples of internal collections
+        include shadow "index" collections
 
         :return:
         """
@@ -462,6 +502,45 @@ class Collection:
         """
         Attach an index to the collection.
 
+        As an example, first let's create a collection in a database:
+
+        >>> from linkml_store import Client
+        >>> from linkml_store.utils.format_utils import load_objects
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb")
+        >>> collection = db.create_collection("Country")
+        >>> objs = load_objects("tests/input/countries/countries.jsonl")
+        >>> collection.insert(objs)
+
+        We will create two indexes - one that indexes the whole object
+        (default behavior), the other one indexes the name only
+
+        >>> full_index = get_indexer("simple")
+        >>> full_index.name = "full"
+        >>> name_index = get_indexer("simple", text_template="{name}")
+        >>> name_index.name = "name"
+        >>> collection.attach_indexer(full_index)
+        >>> collection.attach_indexer(name_index)
+
+        Now let's find objects using the full index, using the string "France".
+        We expect the country France to be the top hit, but the score will
+        be less than zero because we did not match all fields in the object.
+
+        >>> qr = collection.search("France", index_name="full")
+        >>> score, top_obj = qr.ranked_rows[0]
+        >>> assert score > 0.1
+        >>> assert score < 0.5
+        >>> top_obj["code"]
+        'FR'
+
+        Now using the name index
+
+        >>> qr = collection.search("France", index_name="name")
+        >>> score, top_obj = qr.ranked_rows[0]
+        >>> assert score > 0.99
+        >>> top_obj["code"]
+        'FR'
+
         :param index:
         :param name:
         :param auto_index: Automatically index all objects in the collection
@@ -497,15 +576,18 @@ class Collection:
 
     def index_objects(self, objs: List[OBJECT], index_name: str, replace=False, **kwargs):
         """
-        Index a list of objects
+        Index a list of objects using a specified index.
+
+        By default, the indexed objects will be stored in a shadow
+        collection in the same database, with additional fields for the index vector
 
         :param objs:
-        :param index_name:
+        :param index_name: e.g. simple, llm
         :param replace:
         :param kwargs:
         :return:
         """
-        ix = self._indexers.get(index_name)
+        ix = self._indexers.get(index_name, None)
         if not ix:
             raise ValueError(f"No index named {index_name}")
         ix_coll_name = self._index_collection_name(index_name)
@@ -523,6 +605,7 @@ class Collection:
                 ix_coll.delete_where()
 
         ix_coll.insert(objects_with_ix, **kwargs)
+        ix_coll.commit()
 
     def list_index_names(self) -> List[str]:
         """
@@ -557,12 +640,22 @@ class Collection:
 
         :return:
         """
-        sv = self.parent.schema_view
+        sv: SchemaView = self.parent.schema_view
         if sv:
             cls = sv.get_class(self.target_class_name)
+            if cls and not cls.attributes:
+                if not sv.class_induced_slots(cls.name):
+                    for att in self._induce_attributes():
+                        cls.attributes[att.name] = att
+                    sv.set_modified()
             return cls
         return None
 
+    def _induce_attributes(self) -> List[SlotDefinition]:
+        result = self.find({}, limit=-1)
+        cd = self.induce_class_definition_from_objects(result.rows, max_sample_size=None)
+        return list(cd.attributes.values())
+
     @property
     def identifier_attribute_name(self) -> Optional[str]:
         """
@@ -579,6 +672,37 @@ class Collection:
                     return att.name
         return None
 
+    def set_identifier_attribute_name(self, name: str):
+        """
+        Set the name of the identifier attribute for the collection.
+
+        AKA the primary key.
+
+        :param name: The name of the identifier attribute.
+        """
+        cd = self.class_definition()
+        if not cd:
+            raise ValueError(f"Cannot find class definition for {self.target_class_name}")
+        id_att = None
+        candidates = []
+        sv: SchemaView = self.parent.schema_view
+        cls = sv.get_class(cd.name)
+        existing_id_slot = sv.get_identifier_slot(cls.name)
+        if existing_id_slot:
+            if existing_id_slot.name == name:
+                return
+            existing_id_slot.identifier = False
+        for att in cls.attributes.values():
+            candidates.append(att.name)
+            if att.name == name:
+                att.identifier = True
+                id_att = att
+            else:
+                att.identifier = False
+        if not id_att:
+            raise ValueError(f"No attribute found with name {name} in {candidates}")
+        sv.set_modified()
+
     def object_identifier(self, obj: OBJECT, auto=True) -> Optional[IDENTIFIER]:
         """
         Return the identifier for an object.
@@ -622,6 +746,8 @@ class Collection:
             for k, v in obj.items():
                 keys[k].append(v)
         for k, vs in keys.items():
+            if k == "_id":
+                continue
             multivalueds = []
             inlineds = []
             rngs = []
@@ -698,6 +824,39 @@ class Collection:
         """
         raise NotImplementedError
 
+    def apply_patches(self, patches: List[PatchDict], **kwargs):
+        """
+        Apply a patch to the collection.
+
+        Patches conform to the JSON Patch format,
+
+        :param patches:
+        :param kwargs:
+        :return:
+        """
+        all_objs = self.find(limit=-1).rows
+        primary_key = self.identifier_attribute_name
+        if not primary_key:
+            raise ValueError(f"No primary key for {self.target_class_name}")
+        new_objs = apply_patches_to_list(all_objs, patches, primary_key=primary_key, **kwargs)
+        self.replace(new_objs)
+
+    def diff(self, other: "Collection", **kwargs):
+        """
+        Diff two collections.
+
+        :param other:
+        :param kwargs:
+        :return:
+        """
+        src_objs = self.find(limit=-1).rows
+        tgt_objs = other.find(limit=-1).rows
+        primary_key = self.identifier_attribute_name
+        if not primary_key:
+            raise ValueError(f"No primary key for {self.target_class_name}")
+        patches_from_objects_lists(src_objs, tgt_objs, primary_key=primary_key)
+        return patches_from_objects_lists(src_objs, tgt_objs, primary_key=primary_key)
+
     def iter_validate_collection(self, **kwargs) -> Iterator["ValidationResult"]:
         """
         Validate the contents of the collection
@@ -717,3 +876,14 @@ class Collection:
         for obj in result.rows:
             obj = clean_empties(obj)
             yield from validator.iter_results(obj, class_name)
+
+    def commit(self):
+        """
+        Commit changes to the collection.
+
+        :return:
+        """
+        pass
+
+    def _broadcast(self, *args, **kwargs):
+        self.parent.broadcast(self, *args, **kwargs)

linkml_store/api/config.py

@@ -3,7 +3,11 @@ from typing import Any, Dict, List, Optional
 from pydantic import BaseModel, Field
 
 
-class CollectionConfig(BaseModel):
+class ConfiguredBaseModel(BaseModel, extra="forbid"):
+    pass
+
+
+class CollectionConfig(ConfiguredBaseModel):
     name: Optional[str] = Field(
         default=None,
         description="An optional name for the collection",
@@ -42,7 +46,7 @@ class CollectionConfig(BaseModel):
     )
 
 
-class DatabaseConfig(BaseModel):
+class DatabaseConfig(ConfiguredBaseModel):
     handle: str = Field(
         default="duckdb:///:memory:",
         description="The database handle, e.g., 'duckdb:///:memory:' or 'mongodb://localhost:27017'",
@@ -86,7 +90,7 @@ class DatabaseConfig(BaseModel):
     )
 
 
-class ClientConfig(BaseModel):
+class ClientConfig(ConfiguredBaseModel):
     handle: Optional[str] = Field(
         default=None,
         description="The client handle",
@@ -95,6 +99,10 @@ class ClientConfig(BaseModel):
         default={},
         description="A dictionary of database configurations",
     )
+    default_database: Optional[str] = Field(
+        default=None,
+        description="The default database",
+    )
     schema_path: Optional[str] = Field(
         default=None,
         description="The path to the LinkML schema file",

linkml_store/api/database.py

@@ -3,9 +3,24 @@ from abc import ABC
 from collections import defaultdict
 from copy import copy
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterator, Optional, Sequence, Type, Union
-
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    ClassVar,
+    Dict,
+    Generic,
+    Iterator,
+    List,
+    Optional,
+    Sequence,
+    Type,
+    Union,
+)
+
+from linkml_store.api.types import CollectionType
 from linkml_store.utils.format_utils import load_objects, render_output
+from linkml_store.utils.patch_utils import PatchDict
 
 try:
     from linkml.validator.report import Severity, ValidationResult
@@ -24,8 +39,10 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+LISTENER = Callable[[Collection, List[PatchDict]], None]
+
 
-class Database(ABC):
+class Database(ABC, Generic[CollectionType]):
     """
     A Database provides access to named collections of data.
 
@@ -89,6 +106,8 @@ class Database(ABC):
     metadata: Optional[DatabaseConfig] = None
     collection_class: ClassVar[Optional[Type[Collection]]] = None
 
+    listeners: Optional[List[LISTENER]] = None
+
     def __init__(self, handle: Optional[str] = None, metadata: Optional[DatabaseConfig] = None, **kwargs):
         if metadata:
             self.metadata = metadata
@@ -233,7 +252,8 @@ class Database(ABC):
         :param kwargs:
         :return:
         """
-        raise NotImplementedError()
+        for coll in self.list_collections():
+            coll.commit()
 
     def close(self, **kwargs):
         """
@@ -301,6 +321,7 @@ class Database(ABC):
             alias = name
         self._collections[alias] = collection
         if recreate_if_exists:
+            logger.debug(f"Recreating collection {collection.name}")
             collection.delete_where({}, missing_ok=True)
         return collection
 
@@ -418,7 +439,11 @@ class Database(ABC):
         :return:
 
         """
-        raise NotImplementedError
+        if query.from_table:
+            collection = self.get_collection(query.from_table)
+            return collection.query(query, **kwargs)
+        else:
+            raise NotImplementedError(f"Querying without a table is not supported in {self.__class__.__name__}")
 
     @property
     def schema_view(self) -> SchemaView:
@@ -689,3 +714,9 @@ class Database(ABC):
         logger.info(f"Exporting object with {len(obj)} collections to {location} in {target_format} format")
         with open(location, "w", encoding="utf-8") as stream:
             stream.write(render_output(obj, format=target_format))
+
+    def broadcast(self, source: Collection, patches: List[PatchDict]):
+        if not self.listeners:
+            return
+        for listener in self.listeners:
+            listener(source, patches)

linkml_store/api/stores/duckdb/duckdb_collection.py

@@ -38,6 +38,7 @@ class DuckDBCollection(Collection):
             with conn.begin():
                 conn.execute(insert(table), objs)
             conn.commit()
+        self._post_insert_hook(objs)
 
     def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> Optional[int]:
         if not isinstance(objs, list):
@@ -89,7 +90,9 @@ class DuckDBCollection(Collection):
         cd = self.class_definition()
         with self.parent.engine.connect() as conn:
             if not facet_columns:
-                facet_columns = list(self.class_definition().attributes.keys())
+                if not cd:
+                    raise ValueError(f"No class definition found for {self.target_class_name}")
+                facet_columns = list(cd.attributes.keys())
             for col in facet_columns:
                 logger.debug(f"Faceting on {col}")
                 if isinstance(col, tuple):
@@ -100,7 +103,7 @@ class DuckDBCollection(Collection):
                 facet_query_str = facet_count_sql(facet_query, col, multivalued=sd.multivalued)
                 logger.debug(f"Facet query: {facet_query_str}")
                 rows = list(conn.execute(text(facet_query_str)))
-                results[col] = rows
+                results[col] = [tuple(row) for row in rows]
             return results
 
     def _sqla_table(self, cd: ClassDefinition) -> Table:
@@ -109,7 +112,7 @@ class DuckDBCollection(Collection):
         cols = []
         for att in schema_view.class_induced_slots(cd.name):
             typ = TMAP.get(att.range, sqla.String)
-            if att.inlined:
+            if att.inlined or att.inlined_as_list:
                 typ = sqla.JSON
             if att.multivalued:
                 typ = sqla.ARRAY(typ, dimensions=1)

linkml_store/api/stores/duckdb/duckdb_database.py

@@ -31,6 +31,18 @@ logger = logging.getLogger(__name__)
 
 
 class DuckDBDatabase(Database):
+    """
+    An adapter for DuckDB databases.
+
+    Note that this adapter does not make use of a LinkML relational model transformation and
+    SQL Alchemy ORM layer. Instead, it attempts to map each collection (which is of type
+    some LinkML class) to a *single* DuckDB table. New tables are not created for nested references,
+    and linking tables are not created for many-to-many relationships.
+
+    Instead the native DuckDB ARRAY type is used to store multivalued attributes, and DuckDB JSON
+    types are used for nested inlined objects.
+    """
+
     _connection: DuckDBPyConnection = None
     _engine: sqlalchemy.Engine = None
     collection_class = DuckDBCollection
@@ -103,7 +115,14 @@ class DuckDBDatabase(Database):
                         if row[col]:
                             if isinstance(row[col], list):
                                 for i in range(len(row[col])):
-                                    row[col][i] = json.loads(row[col][i])
+                                    try:
+                                        parsed_val = json.loads(row[col][i])
+                                    except json.JSONDecodeError as e:
+                                        logger.error(f"Failed to parse col {col}[{i}] == {row[col][i]}")
+                                        raise e
+                                    row[col][i] = parsed_val
+                            elif isinstance(row[col], dict):
+                                pass
                             else:
                                 row[col] = json.loads(row[col])
             qr.set_rows(pd.DataFrame(rows))

linkml_store/api/stores/filesystem/__init__.py

@@ -1,16 +1,15 @@
 """
-Adapter for DuckDB embedded database.
+Adapter for FileSystem wrapper
 
 Handles have the form:
 
- - ``duckdb:///<path>`` for a file-based database
- - ``duckdb:///:memory:`` for an in-memory database
-"""
+ - ``file:<path>`` for a local file
+ """
 
-from linkml_store.api.stores.duckdb.duckdb_collection import DuckDBCollection
-from linkml_store.api.stores.duckdb.duckdb_database import DuckDBDatabase
+from linkml_store.api.stores.filesystem.filesystem_collection import FileSystemCollection
+from linkml_store.api.stores.filesystem.filesystem_database import FileSystemDatabase
 
 __all__ = [
-    "DuckDBCollection",
-    "DuckDBDatabase",
+    "FileSystemCollection",
+    "FileSystemDatabase",
 ]