linkml-store 0.0.0__py3-none-any.whl → 0.1.6__py3-none-any.whl

linkml_store/api/__init__.py

@@ -1,8 +1,8 @@
 # flake8: noqa: E402
 from linkml_store.api.collection import Collection
 from linkml_store.api.database import Database
-from linkml_store.api.metadata import MetaData
 from linkml_store.api.client import Client
+
 # flake8: noqa
 
-__all__ = ["Client", "Database", "MetaData", "Collection"]
+__all__ = ["Client", "Database", "Collection"]

linkml_store/api/client.py

@@ -1,17 +1,24 @@
-from dataclasses import dataclass
-from typing import Dict, Optional
+from pathlib import Path
+from typing import Dict, Optional, Union
 
+import yaml
 from linkml_runtime import SchemaView
 
 from linkml_store.api import Database
+from linkml_store.api.config import ClientConfig
+from linkml_store.api.stores.chromadb.chromadb_database import ChromaDBDatabase
 from linkml_store.api.stores.duckdb.duckdb_database import DuckDBDatabase
+from linkml_store.api.stores.mongodb.mongodb_database import MongoDBDatabase
+from linkml_store.api.stores.solr.solr_database import SolrDatabase
 
 HANDLE_MAP = {
     "duckdb": DuckDBDatabase,
+    "solr": SolrDatabase,
+    "mongodb": MongoDBDatabase,
+    "chromadb": ChromaDBDatabase,
 }
 
 
-@dataclass
 class Client:
     """
     A client provides access to named collections.
@@ -22,7 +29,7 @@ class Client:
     >>> db = client.attach_database("duckdb", alias="test")
     >>> collection = db.create_collection("Person")
     >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
-    >>> collection.add(objs)
+    >>> collection.insert(objs)
     >>> qr = collection.find()
     >>> len(qr.rows)
     2
@@ -38,9 +45,76 @@ class Client:
 
     """
 
-    handle: Optional[str] = None
+    metadata: Optional[ClientConfig] = None
     _databases: Optional[Dict[str, Database]] = None
 
+    def __init__(self, handle: Optional[str] = None, metadata: Optional[ClientConfig] = None):
+        """
+        Initialize a client.
+
+        :param handle:
+        :param metadata:
+        """
+        self.metadata = metadata
+        if not self.metadata:
+            self.metadata = ClientConfig()
+        self.metadata.handle = handle
+
+    @property
+    def handle(self) -> Optional[str]:
+        return self.metadata.handle
+
+    @property
+    def base_dir(self) -> Optional[str]:
+        """
+        Get the base directory for the client.
+
+        Wraps metadata.base_dir.
+
+        :return:
+        """
+        return self.metadata.base_dir
+
+    def from_config(self, config: Union[ClientConfig, str, Path], base_dir=None, **kwargs):
+        """
+        Create a client from a configuration.
+
+        Examples
+        --------
+        >>> from linkml_store.api.config import ClientConfig
+        >>> client = Client().from_config(ClientConfig(databases={"test": {"handle": "duckdb:///:memory:"}}))
+        >>> len(client.databases)
+        1
+        >>> "test" in client.databases
+        True
+        >>> client.databases["test"].handle
+        'duckdb:///:memory:'
+
+        :param config:
+        :param kwargs:
+        :return:
+
+        """
+        if isinstance(config, Path):
+            config = str(config)
+        if isinstance(config, str):
+            if not base_dir:
+                base_dir = Path(config).parent
+            parsed_obj = yaml.safe_load(open(config))
+            config = ClientConfig(**parsed_obj)
+        self.metadata = config
+        if base_dir:
+            self.metadata.base_dir = base_dir
+        self._initialize_databases(**kwargs)
+        return self
+
+    def _initialize_databases(self, **kwargs):
+        for name, db_config in self.metadata.databases.items():
+            handle = db_config.handle.format(base_dir=self.base_dir)
+            db_config.handle = handle
+            db = self.attach_database(handle, alias=name, **kwargs)
+            db.from_config(db_config)
+
     def attach_database(
         self,
         handle: str,
@@ -69,7 +143,6 @@ class Client:
         :param schema_view: schema view to associate with the database
         :param kwargs:
         :return:
-
         """
         if ":" not in handle:
             scheme = handle
@@ -87,6 +160,7 @@ class Client:
         if not self._databases:
             self._databases = {}
         self._databases[alias] = db
+        db.parent = self
         return db
 
     def get_database(self, name: Optional[str] = None, create_if_not_exists=True, **kwargs) -> Database:
@@ -101,7 +175,7 @@ class Client:
         >>> db == retrieved_db
         True
 
-        :param name:
+        :param name: if None, there must be a single database attached
         :param create_if_not_exists:
         :param kwargs:
         :return:
@@ -149,3 +223,30 @@ class Client:
         if not self._databases:
             self._databases = {}
         return self._databases
+
+    def drop_database(self, name: str, missing_ok=False, **kwargs):
+        """
+        Drop a database.
+
+        :param name:
+        :param missing_ok:
+        :return:
+        """
+        if name in self._databases:
+            db = self._databases[name]
+            db.drop(**kwargs)
+            del self._databases[name]
+        else:
+            if not missing_ok:
+                raise ValueError(f"Database {name} not found")
+
+    def drop_all_databases(self, **kwargs):
+        """
+        Drop all databases.
+
+        :param missing_ok:
+        :return:
+        """
+        for name in list(self._databases.keys()):
+            self.drop_database(name, missing_ok=False, **kwargs)
+        self._databases = {}

linkml_store/api/collection.py

@@ -1,16 +1,24 @@
+import hashlib
 import logging
 from collections import defaultdict
-from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, TextIO, Type, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, TextIO, Type, Union
 
 import numpy as np
 from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition
 from linkml_runtime.linkml_model.meta import ArrayExpression
 from pydantic import BaseModel
 
+from linkml_store.index import get_indexer
+
+try:
+    from linkml.validator.report import ValidationResult
+except ImportError:
+    ValidationResult = None
+
+from linkml_store.api.config import CollectionConfig
 from linkml_store.api.queries import Query, QueryResult
-from linkml_store.index.index import Index
+from linkml_store.index.indexer import Indexer
 
 if TYPE_CHECKING:
     from linkml_store.api.database import Database
@@ -19,11 +27,11 @@ logger = logging.getLogger(__name__)
 
 OBJECT = Union[Dict[str, Any], BaseModel, Type]
 
+DEFAULT_FACET_LIMIT = 100
 IDENTIFIER = str
 FIELD_NAME = str
 
 
-@dataclass
 class Collection:
     """
     A collection is an organized set of objects of the same or similar type.
@@ -33,12 +41,80 @@ class Collection:
     - For a file system, a collection could be a single tabular file such as Parquet or CSV
     """
 
-    name: str
+    # name: str
     parent: Optional["Database"] = None
-    _indexes: Optional[Dict[str, Index]] = None
-    hidden: Optional[bool] = False
+    _indexers: Optional[Dict[str, Indexer]] = None
+    # hidden: Optional[bool] = False
+
+    metadata: Optional[CollectionConfig] = None
+
+    def __init__(
+        self, name: str, parent: Optional["Database"] = None, metadata: Optional[CollectionConfig] = None, **kwargs
+    ):
+        self.parent = parent
+        if metadata:
+            self.metadata = metadata
+        else:
+            self.metadata = CollectionConfig(name=name, **kwargs)
+        if name is not None and self.metadata.name is not None and name != self.metadata.name:
+            raise ValueError(f"Name mismatch: {name} != {self.metadata.name}")
+
+    @property
+    def name(self) -> str:
+        return self.metadata.name
+
+    @property
+    def hidden(self) -> bool:
+        return self.metadata.hidden
+
+    @property
+    def _target_class_name(self):
+        """
+        Return the name of the class that this collection represents
+
+        This MUST be a LinkML class name
+
+        :return:
+        """
+        # TODO: this is a shim layer until we can normalize on this
+        if self.metadata.type:
+            return self.metadata.type
+        return self.name
 
-    def add(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
+    @property
+    def _alias(self):
+        """
+        Return the primary name/alias used for the collection.
+
+        This MAY be the name of the LinkML class, but it may be desirable
+        to have an alias, for example "persons" which collects all instances
+        of class Person.
+
+        The _alias SHOULD be used for Table names in SQL.
+
+        For nested data, the alias SHOULD be used as the key; e.g
+
+         ``{ "persons": [ { "name": "Alice" }, { "name": "Bob" } ] }``
+
+        :return:
+        """
+        # TODO: this is a shim layer until we can normalize on this
+        if self.metadata.alias:
+            return self.metadata.alias
+        return self.name
+
+    def replace(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
+        """
+        Replace entire collection with objects.
+
+        :param objs:
+        :param kwargs:
+        :return:
+        """
+        self.delete_where({})
+        self.insert(objs, **kwargs)
+
+    def insert(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         """
         Add one or more objects to the collection
 
@@ -58,13 +134,14 @@ class Collection:
         """
         raise NotImplementedError
 
-    def delete_where(self, where: Optional[Dict[str, Any]] = None, **kwargs) -> int:
+    def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> int:
         """
         Delete objects that match a query
 
-        :param where:
+        :param where: where conditions
+        :param missing_ok: if True, do not raise an error if the collection does not exist
         :param kwargs:
-        :return:
+        :return: number of objects deleted (or -1 if unsupported)
         """
         raise NotImplementedError
 
@@ -79,7 +156,7 @@ class Collection:
         raise NotImplementedError
 
     def _create_query(self, **kwargs) -> Query:
-        return Query(from_table=self.name, **kwargs)
+        return Query(from_table=self._alias, **kwargs)
 
     def query(self, query: Query, **kwargs) -> QueryResult:
         """
@@ -91,7 +168,9 @@ class Collection:
         """
         return self.parent.query(query, **kwargs)
 
-    def query_facets(self, where: Optional[Dict] = None, facet_columns: List[str] = None) -> Dict[str, Dict[str, int]]:
+    def query_facets(
+        self, where: Optional[Dict] = None, facet_columns: List[str] = None, facet_limit=DEFAULT_FACET_LIMIT, **kwargs
+    ) -> Dict[str, Dict[str, int]]:
         """
         Run a query to get facet counts for one or more columns.
 
@@ -108,17 +187,32 @@ class Collection:
         :param con: A DuckDB database connection.
         :param query: A Query object representing the base query.
         :param facet_columns: A list of column names to get facet counts for.
+        :param facet_limit:
         :return: A dictionary where keys are column names and values are pandas DataFrames
                  containing the facet counts for each unique value in the respective column.
         """
         raise NotImplementedError
 
     def get(self, ids: Optional[IDENTIFIER], **kwargs) -> QueryResult:
+        """
+        Get one or more objects by ID.
+
+        :param ids:
+        :param kwargs:
+        :return:
+        """
         id_field = self.identifier_field
         q = self._create_query(where_clause={id_field: ids})
         return self.query(q, **kwargs)
 
     def find(self, where: Optional[Any] = None, **kwargs) -> QueryResult:
+        """
+        Find objects in the collection using a where query.
+
+        :param where:
+        :param kwargs:
+        :return:
+        """
         query = self._create_query(where_clause=where)
         return self.query(query, **kwargs)
 
@@ -141,66 +235,122 @@ class Collection:
         :return:
         """
         if index_name is None:
-            if len(self._indexes) == 1:
-                index_name = list(self._indexes.keys())[0]
+            if len(self._indexers) == 1:
+                index_name = list(self._indexers.keys())[0]
             else:
                 raise ValueError("Multiple indexes found. Please specify an index name.")
         ix_coll = self.parent.get_collection(self._index_collection_name(index_name))
-        ix = self._indexes.get(index_name)
+        ix = self._indexers.get(index_name)
         if not ix:
             raise ValueError(f"No index named {index_name}")
         qr = ix_coll.find(where=where, limit=-1, **kwargs)
         index_col = ix.index_field
         vector_pairs = [(row, np.array(row[index_col], dtype=float)) for row in qr.rows]
         results = ix.search(query, vector_pairs, limit=limit)
+        for r in results:
+            del r[1][index_col]
         new_qr = QueryResult(num_rows=len(results))
         new_qr.ranked_rows = results
         return new_qr
 
-    def attach_index(self, index: Index, auto_index=True, **kwargs):
+    @property
+    def is_internal(self) -> bool:
+        """
+        Check if the collection is internal
+
+        :return:
+        """
+        if not self.name:
+            raise ValueError(f"Collection has no name: {self} // {self.metadata}")
+        return self.name.startswith("internal__")
+
+    def attach_indexer(self, index: Union[Indexer, str], name: Optional[str] = True, auto_index=True, **kwargs):
         """
         Attach an index to the collection.
 
         :param index:
-        :param auto_index:
+        :param name:
+        :param auto_index: Automatically index all objects in the collection
         :param kwargs:
         :return:
         """
+        if isinstance(index, str):
+            index = get_indexer(index)
+        if name:
+            index.name = name
+        if not index.name:
+            index.name = type(index).__name__.lower()
         index_name = index.name
         if not index_name:
             raise ValueError("Index must have a name")
-        if not self._indexes:
-            self._indexes = {}
-        self._indexes[index_name] = index
+        if not self._indexers:
+            self._indexers = {}
+        self._indexers[index_name] = index
         if auto_index:
             all_objs = self.find(limit=-1).rows
-            self.index_objects(all_objs, index_name, **kwargs)
+            self.index_objects(all_objs, index_name, replace=True, **kwargs)
 
     def _index_collection_name(self, index_name: str) -> str:
-        return f"index__{self.name}_{index_name}"
+        """
+        Create a name for a special collection that holds index data
+
+        :param index_name:
+        :return:
+        """
+        return f"internal__index__{self.name}__{index_name}"
 
-    def index_objects(self, objs: List[OBJECT], index_name: str, **kwargs):
+    def index_objects(self, objs: List[OBJECT], index_name: str, replace=False, **kwargs):
         """
         Index a list of objects
 
         :param objs:
         :param index_name:
+        :param replace:
         :param kwargs:
         :return:
         """
-        ix = self._indexes.get(index_name)
+        ix = self._indexers.get(index_name)
         if not ix:
             raise ValueError(f"No index named {index_name}")
-        ix_coll = self.parent.get_collection(self._index_collection_name(index_name), create_if_not_exists=True)
+        ix_coll_name = self._index_collection_name(index_name)
+        ix_coll = self.parent.get_collection(ix_coll_name, create_if_not_exists=True)
         vectors = [list(float(e) for e in v) for v in ix.objects_to_vectors(objs)]
         objects_with_ix = []
         index_col = ix.index_field
         for obj, vector in zip(objs, vectors):
             # TODO: id field
             objects_with_ix.append({**obj, **{index_col: vector}})
-        ix_coll.add(objects_with_ix, **kwargs)
+        if replace:
+            schema = self.parent.schema_view.schema
+            logger.info(f"Checking if {ix_coll_name} is in {schema.classes.keys()}")
+            if ix_coll_name in schema.classes:
+                ix_coll.delete_where()
+        ix_coll.insert(objects_with_ix, **kwargs)
+
+    def list_index_names(self) -> List[str]:
+        """
+        Return a list of index names
+
+        :return:
+        """
+        return list(self._indexers.keys())
+
+    @property
+    def indexers(self) -> Dict[str, Indexer]:
+        """
+        Return a list of indexers
+
+        :return:
+        """
+        return self._indexers if self._indexers else {}
 
     def peek(self, limit: Optional[int] = None) -> QueryResult:
+        """
+        Return the first N objects in the collection
+
+        :param limit:
+        :return:
+        """
         q = self._create_query()
         return self.query(q, limit=limit)
 
@@ -212,13 +362,16 @@ class Collection:
         """
         sv = self.parent.schema_view
         if sv:
-            return sv.get_class(self.name)
+            cls = sv.get_class(self._target_class_name)
+            return cls
         return None
 
     def identifier_attribute_name(self) -> Optional[str]:
         """
         Return the name of the identifier attribute for the collection.
 
+        AKA the primary key.
+
         :return: The name of the identifier attribute, if one exists.
         """
         cd = self.class_definition()
@@ -228,6 +381,25 @@ class Collection:
                     return att.name
         return None
 
+    def object_identifier(self, obj: OBJECT, auto=True) -> Optional[IDENTIFIER]:
+        """
+        Return the identifier for an object.
+
+        :param obj:
+        :param auto: If True, generate an identifier if one does not exist.
+        :return:
+        """
+        pk = self.identifier_attribute_name
+        if pk in obj:
+            return obj[pk]
+        elif auto:
+            # TODO: use other unique keys if no primary key
+            as_str = str(obj)
+            md5 = hashlib.md5(as_str.encode()).hexdigest()
+            return md5
+        else:
+            return None
+
     def induce_class_definition_from_objects(self, objs: List[OBJECT], max_sample_size=10) -> ClassDefinition:
         """
         Induce a class definition from a list of objects.
@@ -239,7 +411,7 @@ class Collection:
         :param max_sample_size:
         :return:
         """
-        cd = ClassDefinition(self.name)
+        cd = ClassDefinition(self._target_class_name)
         keys = defaultdict(list)
         for obj in objs[0:max_sample_size]:
             if isinstance(obj, BaseModel):
@@ -302,7 +474,7 @@ class Collection:
                 array_expr = ArrayExpression(exact_number_dimensions=len(exact_dimensions_list[0]))
                 cd.attributes[k].array = array_expr
         sv = self.parent.schema_view
-        sv.schema.classes[self.name] = cd
+        sv.schema.classes[self._target_class_name] = cd
         sv.set_modified()
         return cd
 
@@ -325,3 +497,22 @@ class Collection:
         :return:
         """
         raise NotImplementedError
+
+    def iter_validate_collection(self, **kwargs) -> Iterator["ValidationResult"]:
+        """
+        Validate the contents of the collection
+
+        :param kwargs:
+        :return: iterator over validation results
+        """
+        from linkml.validator import JsonschemaValidationPlugin, Validator
+
+        validation_plugins = [JsonschemaValidationPlugin(closed=True)]
+        validator = Validator(self.parent.schema_view.schema, validation_plugins=validation_plugins)
+        cd = self.class_definition()
+        if not cd:
+            raise ValueError(f"Cannot find class definition for {self._target_class_name}")
+        class_name = cd.name
+        result = self.find(**kwargs)
+        for obj in result.rows:
+            yield from validator.iter_results(obj, class_name)

linkml_store/api/config.py

@@ -0,0 +1,97 @@
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class CollectionConfig(BaseModel):
+    name: Optional[str] = Field(
+        default=None,
+        description="An optional name for the collection",
+    )
+    alias: Optional[str] = Field(
+        default=None,
+        description="An optional alias for the collection",
+    )
+    type: Optional[str] = Field(
+        default=None,
+        description="The type of object in the collection. TODO; use this instead of name",
+    )
+    metadata: Optional[Dict] = Field(
+        default=None,
+        description="Optional metadata for the collection",
+    )
+    attributes: Optional[Dict[str, Dict]] = Field(
+        default=None,
+        description="Optional attributes for the collection, following LinkML schema",
+    )
+    indexers: Optional[Dict[str, Dict]] = Field(
+        default=None,
+        description="Optional configuration for indexers",
+    )
+    hidden: Optional[bool] = Field(
+        default=False,
+        description="Whether the collection is hidden",
+    )
+    is_prepopulated: Optional[bool] = Field(
+        default=False,
+        description="Whether the collection is prepopulated",
+    )
+
+
+class DatabaseConfig(BaseModel):
+    handle: str = Field(
+        default="duckdb:///:memory:",
+        description="The database handle, e.g., 'duckdb:///:memory:' or 'mongodb://localhost:27017'",
+    )
+    alias: Optional[str] = Field(
+        default=None,
+        description="An optional alias for the database",
+    )
+    schema_location: Optional[str] = Field(
+        default=None,
+        description="The location of the schema file, either a path on disk or URL",
+    )
+    schema_dict: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description="The LinkML schema as a dictionary",
+    )
+    collections: Dict[str, CollectionConfig] = Field(
+        default={},
+        description="A dictionary of collection configurations",
+    )
+    recreate_if_exists: bool = Field(
+        default=False,
+        description="Whether to recreate the database if it already exists",
+    )
+    collection_type_slot: Optional[str] = Field(
+        default=None,
+        description=(
+            "For databases that combine multiple collections into a single space, this field"
+            "specifies the field that contains the collection type. An example of this is a Solr"
+            "index that does not use cores for collections, and instead uses a single global"
+            "document space; if this has a field 'document_type', then this field should be set"
+        ),
+    )
+    searchable_slots: Optional[List[str]] = Field(
+        default=None,
+        description="Optional configuration for search fields",
+    )
+
+
+class ClientConfig(BaseModel):
+    handle: Optional[str] = Field(
+        default=None,
+        description="The client handle",
+    )
+    databases: Dict[str, DatabaseConfig] = Field(
+        default={},
+        description="A dictionary of database configurations",
+    )
+    schema_path: Optional[str] = Field(
+        default=None,
+        description="The path to the LinkML schema file",
+    )
+    base_dir: Optional[str] = Field(
+        default=None,
+        description="The base directory for the client",
+    )