linkml-store 0.1.9__py3-none-any.whl → 0.1.11__py3-none-any.whl

linkml_store/api/client.py

@@ -98,7 +98,7 @@ class Client:
         """
         return self.metadata.base_dir
 
-    def from_config(self, config: Union[ClientConfig, str, Path], base_dir=None, **kwargs):
+    def from_config(self, config: Union[ClientConfig, dict, str, Path], base_dir=None, **kwargs):
         """
         Create a client from a configuration.
 
@@ -118,11 +118,13 @@ class Client:
         :return:
 
         """
+        if isinstance(config, dict):
+            config = ClientConfig(**config)
         if isinstance(config, Path):
             config = str(config)
         if isinstance(config, str):
-            if not base_dir:
-                base_dir = Path(config).parent
+            # if not base_dir:
+            #    base_dir = Path(config).parent
             parsed_obj = yaml.safe_load(open(config))
             config = ClientConfig(**parsed_obj)
         self.metadata = config
@@ -133,8 +135,15 @@ class Client:
 
     def _initialize_databases(self, **kwargs):
         for name, db_config in self.metadata.databases.items():
-            handle = db_config.handle.format(base_dir=self.base_dir)
+            base_dir = self.base_dir
+            logger.info(f"Initializing database: {name}, base_dir: {base_dir}")
+            if not base_dir:
+                base_dir = Path.cwd()
+                logger.info(f"Using current working directory: {base_dir}")
+            handle = db_config.handle.format(base_dir=base_dir)
             db_config.handle = handle
+            if db_config.schema_location:
+                db_config.schema_location = db_config.schema_location.format(base_dir=base_dir)
             db = self.attach_database(handle, alias=name, **kwargs)
             db.from_config(db_config)
 
@@ -233,7 +242,7 @@ class Client:
         Return all attached databases
 
         Examples
-        --------
+
         >>> client = Client()
         >>> _ = client.attach_database("duckdb", alias="test1")
         >>> _ = client.attach_database("duckdb", alias="test2")
@@ -259,25 +268,81 @@ class Client:
         """
         Drop a database.
 
+        Example (in-memory):
+
+        >>> client = Client()
+        >>> db1 = client.attach_database("duckdb", alias="test1")
+        >>> db2 = client.attach_database("duckdb", alias="test2")
+        >>> len(client.databases)
+        2
+        >>> client.drop_database("test1")
+        >>> len(client.databases)
+        1
+
+        Databases that persist on disk:
+
+        >>> client = Client()
+        >>> path = Path("tmp/test.db")
+        >>> path.parent.mkdir(parents=True, exist_ok=True)
+        >>> db = client.attach_database(f"duckdb:///{path}", alias="test")
+        >>> len(client.databases)
+        1
+        >>> db.store({"persons": [{"id": "P1", "name": "John"}]})
+        >>> db.commit()
+        >>> Path("tmp/test.db").exists()
+        True
+        >>> client.drop_database("test")
+        >>> len(client.databases)
+        0
+        >>> Path("tmp/test.db").exists()
+        False
+
+        Dropping a non-existent database:
+
+        >>> client = Client()
+        >>> client.drop_database("duckdb:///tmp/made-up1", missing_ok=True)
+        >>> client.drop_database("duckdb:///tmp/made-up2", missing_ok=False)
+        Traceback (most recent call last):
+        ...
+        ValueError: Database duckdb:///tmp/made-up2 not found
+
         :param name:
         :param missing_ok:
         :return:
         """
-        if name in self._databases:
-            db = self._databases[name]
-            db.drop(**kwargs)
-            del self._databases[name]
+        if self._databases:
+            if name in self._databases:
+                db = self._databases[name]
+                db.drop(**kwargs)
+                del self._databases[name]
+            else:
+                if not missing_ok:
+                    raise ValueError(f"Database {name} not found")
         else:
-            if not missing_ok:
-                raise ValueError(f"Database {name} not found")
+            db = self.get_database(name, create_if_not_exists=True)
+            db.drop(**kwargs)
 
     def drop_all_databases(self, **kwargs):
         """
         Drop all databases.
 
+        Example (in-memory):
+
+        >>> client = Client()
+        >>> db1 = client.attach_database("duckdb", alias="test1")
+        >>> assert "test1" in client.databases
+        >>> db2 = client.attach_database("duckdb", alias="test2")
+        >>> assert "test2" in client.databases
+        >>> client.drop_all_databases()
+        >>> len(client.databases)
+        0
+
+
         :param missing_ok:
         :return:
         """
+        if not self._databases:
+            return
         for name in list(self._databases.keys()):
             self.drop_database(name, missing_ok=False, **kwargs)
         self._databases = {}

linkml_store/api/collection.py

@@ -4,7 +4,7 @@ import hashlib
 import logging
 from collections import defaultdict
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Generic, Iterator, List, Optional, TextIO, Tuple, Type, Union
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, Generic, Iterator, List, Optional, TextIO, Tuple, Type, Union
 
 import numpy as np
 from linkml_runtime import SchemaView
@@ -14,7 +14,7 @@ from pydantic import BaseModel
 
 from linkml_store.api.types import DatabaseType
 from linkml_store.index import get_indexer
-from linkml_store.utils.format_utils import load_objects
+from linkml_store.utils.format_utils import load_objects, load_objects_from_url
 from linkml_store.utils.object_utils import clean_empties
 from linkml_store.utils.patch_utils import PatchDict, apply_patches_to_list, patches_from_objects_lists
 
@@ -61,9 +61,11 @@ class Collection(Generic[DatabaseType]):
     # name: str
     parent: Optional[DatabaseType] = None
     _indexers: Optional[Dict[str, Indexer]] = None
+    _initialized: Optional[bool] = None
     # hidden: Optional[bool] = False
 
     metadata: Optional[CollectionConfig] = None
+    default_index_name: ClassVar[str] = "simple"
 
     def __init__(
         self, name: str, parent: Optional["Database"] = None, metadata: Optional[CollectionConfig] = None, **kwargs
@@ -72,7 +74,7 @@ class Collection(Generic[DatabaseType]):
         if metadata:
             self.metadata = metadata
         else:
-            self.metadata = CollectionConfig(name=name, **kwargs)
+            self.metadata = CollectionConfig(type=name, **kwargs)
             if not self.metadata.alias:
                 self.metadata.alias = name
             if not self.metadata.type:
@@ -80,17 +82,6 @@ class Collection(Generic[DatabaseType]):
         # if name is not None and self.metadata.name is not None and name != self.metadata.name:
         #    raise ValueError(f"Name mismatch: {name} != {self.metadata.name}")
 
-    @property
-    def name(self) -> str:
-        """
-        Return the name of the collection.
-
-        TODO: deprecate in favor of Type
-
-        :return: name of the collection
-        """
-        return self.metadata.name
-
     @property
     def hidden(self) -> bool:
         """
@@ -117,12 +108,18 @@ class Collection(Generic[DatabaseType]):
         >>> collection.target_class_name
         'Person'
 
+        >>> collection = db.create_collection("Organization")
+        >>> collection.target_class_name
+        'Organization'
+        >>> collection.alias
+        'Organization'
+
         :return: name of the class which members of this collection instantiate
         """
         # TODO: this is a shim layer until we can normalize on this
         if self.metadata.type:
             return self.metadata.type
-        return self.name
+        return self.alias
 
     @property
     def alias(self):
@@ -160,10 +157,9 @@ class Collection(Generic[DatabaseType]):
         :return:
         """
         # TODO: this is a shim layer until we can normalize on this
-        # TODO: this is a shim layer until we can normalize on this
         if self.metadata.alias:
             return self.metadata.alias
-        return self.name
+        return self.target_class_name
 
     def replace(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         """
@@ -200,7 +196,14 @@ class Collection(Generic[DatabaseType]):
         """
         raise NotImplementedError
 
+    def _pre_query_hook(self, query: Optional[Query] = None, **kwargs):
+        logger.info(f"Pre-query hook (state: {self._initialized}; Q= {query}")
+        if not self._initialized:
+            self._materialize_derivations()
+            self._initialized = True
+
     def _post_insert_hook(self, objs: List[OBJECT], **kwargs):
+        self._initialized = True
         patches = [{"op": "add", "path": "/0", "value": obj} for obj in objs]
         self._broadcast(patches, **kwargs)
 
@@ -304,6 +307,7 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
+        self._pre_query_hook()
         return self.parent.query(query, **kwargs)
 
     def query_facets(
@@ -339,7 +343,6 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
-        # TODO
         id_field = self.identifier_attribute_name
         if not id_field:
             raise ValueError(f"No identifier for {self.name}")
@@ -398,9 +401,10 @@ class Collection(Generic[DatabaseType]):
         :return:
         """
         query = self._create_query(where_clause=where)
+        self._pre_query_hook(query)
         return self.query(query, **kwargs)
 
-    def find_iter(self, where: Optional[Any] = None, **kwargs) -> Iterator[OBJECT]:
+    def find_iter(self, where: Optional[Any] = None, page_size=100, **kwargs) -> Iterator[OBJECT]:
         """
         Find objects in the collection using a where query.
 
@@ -408,9 +412,22 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
-        qr = self.find(where=where, limit=-1, **kwargs)
-        for row in qr.rows:
-            yield row
+        total_rows = None
+        offset = 0
+        if page_size < 1:
+            raise ValueError(f"Invalid page size: {page_size}")
+        while True:
+            qr = self.find(where=where, offset=offset, limit=page_size, **kwargs)
+            if total_rows is None:
+                total_rows = qr.num_rows
+            if not qr.rows:
+                return
+            for row in qr.rows:
+                yield row
+            offset += page_size
+            if offset >= total_rows:
+                break
+        return
 
     def search(
         self,
@@ -421,7 +438,30 @@ class Collection(Generic[DatabaseType]):
         **kwargs,
     ) -> QueryResult:
         """
-        Search the collection using a full-text search index.
+        Search the collection using a text-based index index.
+
+        Example:
+
+        >>> from linkml_store import Client
+        >>> from linkml_store.utils.format_utils import load_objects
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb")
+        >>> collection = db.create_collection("Country")
+        >>> objs = load_objects("tests/input/countries/countries.jsonl")
+        >>> collection.insert(objs)
+
+        Now let's index, using the simple trigram-based index
+
+        >>> index = get_indexer("simple")
+        >>> collection.attach_indexer(index)
+
+        Now let's find all objects:
+
+        >>> qr = collection.search("France")
+        >>> score, top_obj = qr.ranked_rows[0]
+        >>> assert score > 0.1
+        >>> top_obj["code"]
+        'FR'
 
         :param query:
         :param where:
@@ -430,13 +470,20 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
+        self._pre_query_hook()
         if index_name is None:
-            if len(self._indexers) == 1:
-                index_name = list(self._indexers.keys())[0]
+            if len(self.indexers) == 1:
+                index_name = list(self.indexers.keys())[0]
             else:
-                raise ValueError("Multiple indexes found. Please specify an index name.")
+                logger.warning("Multiple indexes found. Using default index.")
+                index_name = self.default_index_name
         ix_coll = self.parent.get_collection(self._index_collection_name(index_name))
-        ix = self._indexers.get(index_name)
+        if index_name not in self.indexers:
+            ix = get_indexer(index_name)
+            if not self._indexers:
+                self._indexers = {}
+            self._indexers[index_name] = ix
+        ix = self.indexers.get(index_name)
         if not ix:
             raise ValueError(f"No index named {index_name}")
         qr = ix_coll.find(where=where, limit=-1, **kwargs)
@@ -453,7 +500,10 @@ class Collection(Generic[DatabaseType]):
     @property
     def is_internal(self) -> bool:
         """
-        Check if the collection is internal
+        Check if the collection is internal.
+
+        Internal collections are hidden by default. Examples of internal collections
+        include shadow "index" collections
 
         :return:
         """
@@ -461,14 +511,136 @@ class Collection(Generic[DatabaseType]):
             raise ValueError(f"Collection has no alias: {self} // {self.metadata}")
         return self.alias.startswith("internal__")
 
-    def load_from_source(self):
-        objects = load_objects(self.metadata.source_location)
+    def exists(self) -> Optional[bool]:
+        """
+        Check if the collection exists.
+
+        :return:
+        """
+        cd = self.class_definition()
+        return cd is not None
+
+    def load_from_source(self, load_if_exists=False):
+        """
+        Load objects from the source location.
+
+        :param load_if_exists:
+        :return:
+        """
+        if not load_if_exists and self.exists():
+            return
+        metadata = self.metadata
+        if metadata.source:
+            source = metadata.source
+            kwargs = source.arguments or {}
+            if source.local_path:
+                objects = load_objects(
+                    metadata.source.local_path, format=source.format, expected_type=source.expected_type, **kwargs
+                )
+            elif metadata.source.url:
+                objects = load_objects_from_url(
+                    metadata.source.url, format=source.format, expected_type=source.expected_type, **kwargs
+                )
         self.insert(objects)
 
+    def _check_if_initialized(self) -> bool:
+        return self._initialized
+
+    def _materialize_derivations(self, **kwargs):
+        metadata = self.metadata
+        if not metadata.derived_from:
+            logger.info(f"No metadata for {self.alias}; no derivations")
+            return
+        if self._check_if_initialized():
+            logger.info(f"Already initialized {self.alias}; no derivations")
+            return
+        parent_db = self.parent
+        client = parent_db.parent
+        # cd = self.class_definition()
+        for derivation in metadata.derived_from:
+            # TODO: optimize this; utilize underlying engine
+            logger.info(f"Deriving from {derivation}")
+            if derivation.database:
+                db = client.get_database(derivation.database)
+            else:
+                db = parent_db
+            if derivation.collection:
+                coll = db.get_collection(derivation.collection)
+            else:
+                coll = self
+            coll.class_definition()
+            source_obj_iter = coll.find_iter(derivation.where or {})
+            mappings = derivation.mappings
+            if not mappings:
+                raise ValueError(f"No mappings for {self.name}")
+            target_class_name = self.target_class_name
+            from linkml_map.session import Session
+
+            session = Session()
+            session.set_source_schema(db.schema_view.schema)
+            session.set_object_transformer(
+                {
+                    "class_derivations": {
+                        target_class_name: {
+                            "populated_from": coll.target_class_name,
+                            "slot_derivations": mappings,
+                        },
+                    }
+                },
+            )
+            logger.debug(f"Session Spec: {session.object_transformer}")
+            tr_objs = []
+            for source_obj in source_obj_iter:
+                tr_obj = session.transform(source_obj, source_type=coll.target_class_name)
+                tr_objs.append(tr_obj)
+            if not tr_objs:
+                raise ValueError(f"No objects derived from {coll.name}")
+            self.insert(tr_objs)
+            self.commit()
+
     def attach_indexer(self, index: Union[Indexer, str], name: Optional[str] = None, auto_index=True, **kwargs):
         """
         Attach an index to the collection.
 
+        As an example, first let's create a collection in a database:
+
+        >>> from linkml_store import Client
+        >>> from linkml_store.utils.format_utils import load_objects
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb")
+        >>> collection = db.create_collection("Country")
+        >>> objs = load_objects("tests/input/countries/countries.jsonl")
+        >>> collection.insert(objs)
+
+        We will create two indexes - one that indexes the whole object
+        (default behavior), the other one indexes the name only
+
+        >>> full_index = get_indexer("simple")
+        >>> full_index.name = "full"
+        >>> name_index = get_indexer("simple", text_template="{name}")
+        >>> name_index.name = "name"
+        >>> collection.attach_indexer(full_index)
+        >>> collection.attach_indexer(name_index)
+
+        Now let's find objects using the full index, using the string "France".
+        We expect the country France to be the top hit, but the score will
+        be less than zero because we did not match all fields in the object.
+
+        >>> qr = collection.search("France", index_name="full")
+        >>> score, top_obj = qr.ranked_rows[0]
+        >>> assert score > 0.1
+        >>> assert score < 0.5
+        >>> top_obj["code"]
+        'FR'
+
+        Now using the name index
+
+        >>> qr = collection.search("France", index_name="name")
+        >>> score, top_obj = qr.ranked_rows[0]
+        >>> assert score > 0.99
+        >>> top_obj["code"]
+        'FR'
+
         :param index:
         :param name:
         :param auto_index: Automatically index all objects in the collection
@@ -500,19 +672,22 @@ class Collection(Generic[DatabaseType]):
         :param indexer:
         :return:
         """
-        return f"internal__index__{self.name}__{index_name}"
+        return f"internal__index__{self.alias}__{index_name}"
 
     def index_objects(self, objs: List[OBJECT], index_name: str, replace=False, **kwargs):
         """
-        Index a list of objects
+        Index a list of objects using a specified index.
+
+        By default, the indexed objects will be stored in a shadow
+        collection in the same database, with additional fields for the index vector
 
         :param objs:
-        :param index_name:
+        :param index_name: e.g. simple, llm
         :param replace:
         :param kwargs:
         :return:
         """
-        ix = self._indexers.get(index_name)
+        ix = self._indexers.get(index_name, None)
         if not ix:
             raise ValueError(f"No index named {index_name}")
         ix_coll_name = self._index_collection_name(index_name)
@@ -563,6 +738,9 @@ class Collection(Generic[DatabaseType]):
         """
         Return the class definition for the collection.
 
+        If no schema has been explicitly set, and the native database does not
+        have a schema, then a schema will be induced from the objects in the collection.
+
         :return:
         """
         sv: SchemaView = self.parent.schema_view
@@ -647,7 +825,9 @@ class Collection(Generic[DatabaseType]):
         else:
             return None
 
-    def induce_class_definition_from_objects(self, objs: List[OBJECT], max_sample_size=10) -> ClassDefinition:
+    def induce_class_definition_from_objects(
+        self, objs: List[OBJECT], max_sample_size: Optional[int] = None
+    ) -> ClassDefinition:
         """
         Induce a class definition from a list of objects.
 
@@ -658,6 +838,9 @@ class Collection(Generic[DatabaseType]):
         :param max_sample_size:
         :return:
         """
+        # TODO: use schemaview
+        if max_sample_size is None:
+            max_sample_size = 10
         if not self.target_class_name:
             raise ValueError(f"No target_class_name for {self.alias}")
         cd = ClassDefinition(self.target_class_name)
@@ -720,6 +903,7 @@ class Collection(Generic[DatabaseType]):
             for other_rng in rngs:
                 if rng != other_rng:
                     raise ValueError(f"Conflict: {rng} != {other_rng} for {vs}")
+            logger.debug(f"Inducing {k} as {rng} {multivalued} {inlined}")
             cd.attributes[k] = SlotDefinition(k, range=rng, multivalued=multivalued, inlined=inlined)
             if exact_dimensions_list:
                 array_expr = ArrayExpression(exact_number_dimensions=len(exact_dimensions_list[0]))
@@ -753,7 +937,7 @@ class Collection(Generic[DatabaseType]):
         """
         Apply a patch to the collection.
 
-        Patches conform to the JSON Patch format,
+        Patches conform to the JSON Patch format.
 
         :param patches:
         :param kwargs:
@@ -766,11 +950,11 @@ class Collection(Generic[DatabaseType]):
         new_objs = apply_patches_to_list(all_objs, patches, primary_key=primary_key, **kwargs)
         self.replace(new_objs)
 
-    def diff(self, other: "Collection", **kwargs):
+    def diff(self, other: "Collection", **kwargs) -> List[PatchDict]:
         """
         Diff two collections.
 
-        :param other:
+        :param other: The collection to diff against
         :param kwargs:
         :return:
         """
@@ -797,8 +981,7 @@ class Collection(Generic[DatabaseType]):
         if not cd:
             raise ValueError(f"Cannot find class definition for {self.target_class_name}")
         class_name = cd.name
-        result = self.find(**kwargs)
-        for obj in result.rows:
+        for obj in self.find_iter(**kwargs):
             obj = clean_empties(obj)
             yield from validator.iter_results(obj, class_name)
 

linkml_store/api/config.py

@@ -3,11 +3,44 @@ from typing import Any, Dict, List, Optional
 from pydantic import BaseModel, Field
 
 
-class CollectionConfig(BaseModel):
-    name: Optional[str] = Field(
-        default=None,
-        description="An optional name for the collection",
-    )
+class ConfiguredBaseModel(BaseModel, extra="forbid"):
+    """
+    Base class for all configuration models.
+    """
+
+    pass
+
+
+class DerivationConfiguration(ConfiguredBaseModel):
+    """
+    Configuration for a derivation
+    """
+
+    database: Optional[str] = None
+    collection: Optional[str] = None
+    mappings: Optional[Dict[str, Any]] = None
+    where: Optional[Dict[str, Any]] = None
+
+
+class CollectionSource(ConfiguredBaseModel):
+    """
+    Metadata about a source
+    """
+
+    url: Optional[str] = None
+    local_path: Optional[str] = None
+    source_location: Optional[str] = None
+    refresh_interval_days: Optional[float] = None
+    expected_type: Optional[str] = None
+    format: Optional[str] = None
+    arguments: Optional[Dict[str, Any]] = None
+
+
+class CollectionConfig(ConfiguredBaseModel):
+    """
+    Configuration for a collection
+    """
+
     alias: Optional[str] = Field(
         default=None,
         description="An optional alias for the collection",
@@ -36,13 +69,22 @@ class CollectionConfig(BaseModel):
         default=False,
         description="Whether the collection is prepopulated",
     )
-    source_location: Optional[str] = Field(
+    source: Optional[CollectionSource] = Field(
+        default=None,
+        description="Metadata about the source",
+    )
+    # TODO: derived_from
+    derived_from: Optional[List[DerivationConfiguration]] = Field(
         default=None,
-        description="Filesystem or remote URL that stores the data",
+        description="LinkML-Map derivations",
     )
 
 
-class DatabaseConfig(BaseModel):
+class DatabaseConfig(ConfiguredBaseModel):
+    """
+    Configuration for a database
+    """
+
     handle: str = Field(
         default="duckdb:///:memory:",
         description="The database handle, e.g., 'duckdb:///:memory:' or 'mongodb://localhost:27017'",
@@ -86,7 +128,11 @@ class DatabaseConfig(BaseModel):
     )
 
 
-class ClientConfig(BaseModel):
+class ClientConfig(ConfiguredBaseModel):
+    """
+    Configuration for a client
+    """
+
     handle: Optional[str] = Field(
         default=None,
         description="The client handle",
@@ -95,6 +141,10 @@ class ClientConfig(BaseModel):
         default={},
         description="A dictionary of database configurations",
     )
+    default_database: Optional[str] = Field(
+        default=None,
+        description="The default database",
+    )
     schema_path: Optional[str] = Field(
         default=None,
         description="The path to the LinkML schema file",