linkml-store 0.1.10__py3-none-any.whl → 0.1.11__py3-none-any.whl

linkml_store/api/client.py

@@ -242,7 +242,7 @@ class Client:
         Return all attached databases
 
         Examples
-        --------
+
         >>> client = Client()
         >>> _ = client.attach_database("duckdb", alias="test1")
         >>> _ = client.attach_database("duckdb", alias="test2")
@@ -268,25 +268,81 @@ class Client:
         """
         Drop a database.
 
+        Example (in-memory):
+
+        >>> client = Client()
+        >>> db1 = client.attach_database("duckdb", alias="test1")
+        >>> db2 = client.attach_database("duckdb", alias="test2")
+        >>> len(client.databases)
+        2
+        >>> client.drop_database("test1")
+        >>> len(client.databases)
+        1
+
+        Databases that persist on disk:
+
+        >>> client = Client()
+        >>> path = Path("tmp/test.db")
+        >>> path.parent.mkdir(parents=True, exist_ok=True)
+        >>> db = client.attach_database(f"duckdb:///{path}", alias="test")
+        >>> len(client.databases)
+        1
+        >>> db.store({"persons": [{"id": "P1", "name": "John"}]})
+        >>> db.commit()
+        >>> Path("tmp/test.db").exists()
+        True
+        >>> client.drop_database("test")
+        >>> len(client.databases)
+        0
+        >>> Path("tmp/test.db").exists()
+        False
+
+        Dropping a non-existent database:
+
+        >>> client = Client()
+        >>> client.drop_database("duckdb:///tmp/made-up1", missing_ok=True)
+        >>> client.drop_database("duckdb:///tmp/made-up2", missing_ok=False)
+        Traceback (most recent call last):
+        ...
+        ValueError: Database duckdb:///tmp/made-up2 not found
+
         :param name:
         :param missing_ok:
         :return:
         """
-        if name in self._databases:
-            db = self._databases[name]
-            db.drop(**kwargs)
-            del self._databases[name]
+        if self._databases:
+            if name in self._databases:
+                db = self._databases[name]
+                db.drop(**kwargs)
+                del self._databases[name]
+            else:
+                if not missing_ok:
+                    raise ValueError(f"Database {name} not found")
         else:
-            if not missing_ok:
-                raise ValueError(f"Database {name} not found")
+            db = self.get_database(name, create_if_not_exists=True)
+            db.drop(**kwargs)
 
     def drop_all_databases(self, **kwargs):
         """
         Drop all databases.
 
+        Example (in-memory):
+
+        >>> client = Client()
+        >>> db1 = client.attach_database("duckdb", alias="test1")
+        >>> assert "test1" in client.databases
+        >>> db2 = client.attach_database("duckdb", alias="test2")
+        >>> assert "test2" in client.databases
+        >>> client.drop_all_databases()
+        >>> len(client.databases)
+        0
+
+
         :param missing_ok:
         :return:
         """
+        if not self._databases:
+            return
         for name in list(self._databases.keys()):
             self.drop_database(name, missing_ok=False, **kwargs)
         self._databases = {}

linkml_store/api/collection.py

@@ -14,7 +14,7 @@ from pydantic import BaseModel
 
 from linkml_store.api.types import DatabaseType
 from linkml_store.index import get_indexer
-from linkml_store.utils.format_utils import load_objects
+from linkml_store.utils.format_utils import load_objects, load_objects_from_url
 from linkml_store.utils.object_utils import clean_empties
 from linkml_store.utils.patch_utils import PatchDict, apply_patches_to_list, patches_from_objects_lists
 
@@ -61,6 +61,7 @@ class Collection(Generic[DatabaseType]):
     # name: str
     parent: Optional[DatabaseType] = None
     _indexers: Optional[Dict[str, Indexer]] = None
+    _initialized: Optional[bool] = None
     # hidden: Optional[bool] = False
 
     metadata: Optional[CollectionConfig] = None
@@ -73,7 +74,7 @@ class Collection(Generic[DatabaseType]):
         if metadata:
             self.metadata = metadata
         else:
-            self.metadata = CollectionConfig(name=name, **kwargs)
+            self.metadata = CollectionConfig(type=name, **kwargs)
             if not self.metadata.alias:
                 self.metadata.alias = name
             if not self.metadata.type:
@@ -81,17 +82,6 @@ class Collection(Generic[DatabaseType]):
         # if name is not None and self.metadata.name is not None and name != self.metadata.name:
         #    raise ValueError(f"Name mismatch: {name} != {self.metadata.name}")
 
-    @property
-    def name(self) -> str:
-        """
-        Return the name of the collection.
-
-        TODO: deprecate in favor of Type
-
-        :return: name of the collection
-        """
-        return self.metadata.name
-
     @property
     def hidden(self) -> bool:
         """
@@ -118,12 +108,18 @@ class Collection(Generic[DatabaseType]):
         >>> collection.target_class_name
         'Person'
 
+        >>> collection = db.create_collection("Organization")
+        >>> collection.target_class_name
+        'Organization'
+        >>> collection.alias
+        'Organization'
+
         :return: name of the class which members of this collection instantiate
         """
         # TODO: this is a shim layer until we can normalize on this
         if self.metadata.type:
             return self.metadata.type
-        return self.name
+        return self.alias
 
     @property
     def alias(self):
@@ -161,10 +157,9 @@ class Collection(Generic[DatabaseType]):
         :return:
         """
         # TODO: this is a shim layer until we can normalize on this
-        # TODO: this is a shim layer until we can normalize on this
         if self.metadata.alias:
             return self.metadata.alias
-        return self.name
+        return self.target_class_name
 
     def replace(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         """
@@ -201,7 +196,14 @@ class Collection(Generic[DatabaseType]):
         """
         raise NotImplementedError
 
+    def _pre_query_hook(self, query: Optional[Query] = None, **kwargs):
+        logger.info(f"Pre-query hook (state: {self._initialized}; Q= {query}")
+        if not self._initialized:
+            self._materialize_derivations()
+            self._initialized = True
+
     def _post_insert_hook(self, objs: List[OBJECT], **kwargs):
+        self._initialized = True
         patches = [{"op": "add", "path": "/0", "value": obj} for obj in objs]
         self._broadcast(patches, **kwargs)
 
@@ -305,6 +307,7 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
+        self._pre_query_hook()
         return self.parent.query(query, **kwargs)
 
     def query_facets(
@@ -340,7 +343,6 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
-        # TODO
         id_field = self.identifier_attribute_name
         if not id_field:
             raise ValueError(f"No identifier for {self.name}")
@@ -399,9 +401,10 @@ class Collection(Generic[DatabaseType]):
         :return:
         """
         query = self._create_query(where_clause=where)
+        self._pre_query_hook(query)
         return self.query(query, **kwargs)
 
-    def find_iter(self, where: Optional[Any] = None, **kwargs) -> Iterator[OBJECT]:
+    def find_iter(self, where: Optional[Any] = None, page_size=100, **kwargs) -> Iterator[OBJECT]:
         """
         Find objects in the collection using a where query.
 
@@ -409,9 +412,22 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
-        qr = self.find(where=where, limit=-1, **kwargs)
-        for row in qr.rows:
-            yield row
+        total_rows = None
+        offset = 0
+        if page_size < 1:
+            raise ValueError(f"Invalid page size: {page_size}")
+        while True:
+            qr = self.find(where=where, offset=offset, limit=page_size, **kwargs)
+            if total_rows is None:
+                total_rows = qr.num_rows
+            if not qr.rows:
+                return
+            for row in qr.rows:
+                yield row
+            offset += page_size
+            if offset >= total_rows:
+                break
+        return
 
     def search(
         self,
@@ -454,6 +470,7 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
+        self._pre_query_hook()
         if index_name is None:
             if len(self.indexers) == 1:
                 index_name = list(self.indexers.keys())[0]
@@ -494,10 +511,93 @@ class Collection(Generic[DatabaseType]):
             raise ValueError(f"Collection has no alias: {self} // {self.metadata}")
         return self.alias.startswith("internal__")
 
-    def load_from_source(self):
-        objects = load_objects(self.metadata.source_location)
+    def exists(self) -> Optional[bool]:
+        """
+        Check if the collection exists.
+
+        :return:
+        """
+        cd = self.class_definition()
+        return cd is not None
+
+    def load_from_source(self, load_if_exists=False):
+        """
+        Load objects from the source location.
+
+        :param load_if_exists:
+        :return:
+        """
+        if not load_if_exists and self.exists():
+            return
+        metadata = self.metadata
+        if metadata.source:
+            source = metadata.source
+            kwargs = source.arguments or {}
+            if source.local_path:
+                objects = load_objects(
+                    metadata.source.local_path, format=source.format, expected_type=source.expected_type, **kwargs
+                )
+            elif metadata.source.url:
+                objects = load_objects_from_url(
+                    metadata.source.url, format=source.format, expected_type=source.expected_type, **kwargs
+                )
         self.insert(objects)
 
+    def _check_if_initialized(self) -> bool:
+        return self._initialized
+
+    def _materialize_derivations(self, **kwargs):
+        metadata = self.metadata
+        if not metadata.derived_from:
+            logger.info(f"No metadata for {self.alias}; no derivations")
+            return
+        if self._check_if_initialized():
+            logger.info(f"Already initialized {self.alias}; no derivations")
+            return
+        parent_db = self.parent
+        client = parent_db.parent
+        # cd = self.class_definition()
+        for derivation in metadata.derived_from:
+            # TODO: optimize this; utilize underlying engine
+            logger.info(f"Deriving from {derivation}")
+            if derivation.database:
+                db = client.get_database(derivation.database)
+            else:
+                db = parent_db
+            if derivation.collection:
+                coll = db.get_collection(derivation.collection)
+            else:
+                coll = self
+            coll.class_definition()
+            source_obj_iter = coll.find_iter(derivation.where or {})
+            mappings = derivation.mappings
+            if not mappings:
+                raise ValueError(f"No mappings for {self.name}")
+            target_class_name = self.target_class_name
+            from linkml_map.session import Session
+
+            session = Session()
+            session.set_source_schema(db.schema_view.schema)
+            session.set_object_transformer(
+                {
+                    "class_derivations": {
+                        target_class_name: {
+                            "populated_from": coll.target_class_name,
+                            "slot_derivations": mappings,
+                        },
+                    }
+                },
+            )
+            logger.debug(f"Session Spec: {session.object_transformer}")
+            tr_objs = []
+            for source_obj in source_obj_iter:
+                tr_obj = session.transform(source_obj, source_type=coll.target_class_name)
+                tr_objs.append(tr_obj)
+            if not tr_objs:
+                raise ValueError(f"No objects derived from {coll.name}")
+            self.insert(tr_objs)
+            self.commit()
+
     def attach_indexer(self, index: Union[Indexer, str], name: Optional[str] = None, auto_index=True, **kwargs):
         """
         Attach an index to the collection.
@@ -572,7 +672,7 @@ class Collection(Generic[DatabaseType]):
         :param indexer:
         :return:
         """
-        return f"internal__index__{self.name}__{index_name}"
+        return f"internal__index__{self.alias}__{index_name}"
 
     def index_objects(self, objs: List[OBJECT], index_name: str, replace=False, **kwargs):
         """
@@ -638,6 +738,9 @@ class Collection(Generic[DatabaseType]):
         """
         Return the class definition for the collection.
 
+        If no schema has been explicitly set, and the native database does not
+        have a schema, then a schema will be induced from the objects in the collection.
+
         :return:
         """
         sv: SchemaView = self.parent.schema_view
@@ -722,7 +825,9 @@ class Collection(Generic[DatabaseType]):
         else:
             return None
 
-    def induce_class_definition_from_objects(self, objs: List[OBJECT], max_sample_size=10) -> ClassDefinition:
+    def induce_class_definition_from_objects(
+        self, objs: List[OBJECT], max_sample_size: Optional[int] = None
+    ) -> ClassDefinition:
         """
         Induce a class definition from a list of objects.
 
@@ -733,6 +838,9 @@ class Collection(Generic[DatabaseType]):
         :param max_sample_size:
         :return:
         """
+        # TODO: use schemaview
+        if max_sample_size is None:
+            max_sample_size = 10
         if not self.target_class_name:
             raise ValueError(f"No target_class_name for {self.alias}")
         cd = ClassDefinition(self.target_class_name)
@@ -795,6 +903,7 @@ class Collection(Generic[DatabaseType]):
             for other_rng in rngs:
                 if rng != other_rng:
                     raise ValueError(f"Conflict: {rng} != {other_rng} for {vs}")
+            logger.debug(f"Inducing {k} as {rng} {multivalued} {inlined}")
             cd.attributes[k] = SlotDefinition(k, range=rng, multivalued=multivalued, inlined=inlined)
             if exact_dimensions_list:
                 array_expr = ArrayExpression(exact_number_dimensions=len(exact_dimensions_list[0]))
@@ -828,7 +937,7 @@ class Collection(Generic[DatabaseType]):
         """
         Apply a patch to the collection.
 
-        Patches conform to the JSON Patch format,
+        Patches conform to the JSON Patch format.
 
         :param patches:
         :param kwargs:
@@ -841,11 +950,11 @@ class Collection(Generic[DatabaseType]):
         new_objs = apply_patches_to_list(all_objs, patches, primary_key=primary_key, **kwargs)
         self.replace(new_objs)
 
-    def diff(self, other: "Collection", **kwargs):
+    def diff(self, other: "Collection", **kwargs) -> List[PatchDict]:
         """
         Diff two collections.
 
-        :param other:
+        :param other: The collection to diff against
         :param kwargs:
         :return:
         """
@@ -872,8 +981,7 @@ class Collection(Generic[DatabaseType]):
         if not cd:
             raise ValueError(f"Cannot find class definition for {self.target_class_name}")
         class_name = cd.name
-        result = self.find(**kwargs)
-        for obj in result.rows:
+        for obj in self.find_iter(**kwargs):
             obj = clean_empties(obj)
             yield from validator.iter_results(obj, class_name)
 

linkml_store/api/config.py

@@ -4,14 +4,43 @@ from pydantic import BaseModel, Field
 
 
 class ConfiguredBaseModel(BaseModel, extra="forbid"):
+    """
+    Base class for all configuration models.
+    """
+
     pass
 
 
+class DerivationConfiguration(ConfiguredBaseModel):
+    """
+    Configuration for a derivation
+    """
+
+    database: Optional[str] = None
+    collection: Optional[str] = None
+    mappings: Optional[Dict[str, Any]] = None
+    where: Optional[Dict[str, Any]] = None
+
+
+class CollectionSource(ConfiguredBaseModel):
+    """
+    Metadata about a source
+    """
+
+    url: Optional[str] = None
+    local_path: Optional[str] = None
+    source_location: Optional[str] = None
+    refresh_interval_days: Optional[float] = None
+    expected_type: Optional[str] = None
+    format: Optional[str] = None
+    arguments: Optional[Dict[str, Any]] = None
+
+
 class CollectionConfig(ConfiguredBaseModel):
-    name: Optional[str] = Field(
-        default=None,
-        description="An optional name for the collection",
-    )
+    """
+    Configuration for a collection
+    """
+
     alias: Optional[str] = Field(
         default=None,
         description="An optional alias for the collection",
@@ -40,13 +69,22 @@ class CollectionConfig(ConfiguredBaseModel):
         default=False,
         description="Whether the collection is prepopulated",
     )
-    source_location: Optional[str] = Field(
+    source: Optional[CollectionSource] = Field(
         default=None,
-        description="Filesystem or remote URL that stores the data",
+        description="Metadata about the source",
+    )
+    # TODO: derived_from
+    derived_from: Optional[List[DerivationConfiguration]] = Field(
+        default=None,
+        description="LinkML-Map derivations",
     )
 
 
 class DatabaseConfig(ConfiguredBaseModel):
+    """
+    Configuration for a database
+    """
+
     handle: str = Field(
         default="duckdb:///:memory:",
         description="The database handle, e.g., 'duckdb:///:memory:' or 'mongodb://localhost:27017'",
@@ -91,6 +129,10 @@ class DatabaseConfig(ConfiguredBaseModel):
 
 
 class ClientConfig(ConfiguredBaseModel):
+    """
+    Configuration for a client
+    """
+
     handle: Optional[str] = Field(
         default=None,
         description="The client handle",

linkml_store/api/database.py

@@ -149,26 +149,19 @@ class Database(ABC, Generic[CollectionType]):
     def _initialize_collections(self):
         if not self.metadata.collections:
             return
-        for name, collection_config in self.metadata.collections.items():
-            alias = collection_config.alias
-            typ = collection_config.type
-            # if typ and alias is None:
-            #    alias = name
-            # if typ is None:
-            #    typ = name
-            # collection = self.create_collection(
-            #    typ, alias=alias, metadata=collection_config.metadata
-            # )
-            if False and typ is not None:
-                if not alias:
-                    alias = name
-                name = typ
-            if not collection_config.name:
-                collection_config.name = name
-            _collection = self.create_collection(name, alias=alias, metadata=collection_config)
+        for k, collection_config in self.metadata.collections.items():
+            if collection_config.alias:
+                if collection_config.alias != k:
+                    raise ValueError(f"Alias mismatch: {collection_config.alias} != {k}")
+            alias = k
+            typ = collection_config.type or alias
+            _collection = self.create_collection(typ, alias=alias, metadata=collection_config)
+            assert _collection.alias == alias
+            assert _collection.target_class_name == typ
             if collection_config.attributes:
+                # initialize schema
                 sv = self.schema_view
-                cd = ClassDefinition(name, attributes=collection_config.attributes)
+                cd = ClassDefinition(typ, attributes=collection_config.attributes)
                 sv.schema.classes[cd.name] = cd
                 sv.set_modified()
                 # assert collection.class_definition() is not None
@@ -275,7 +268,7 @@ class Database(ABC, Generic[CollectionType]):
         metadata: Optional[CollectionConfig] = None,
         recreate_if_exists=False,
         **kwargs,
-    ) -> Collection:
+    ) -> CollectionType:
         """
         Create a new collection in the current database.
 
@@ -307,8 +300,10 @@ class Database(ABC, Generic[CollectionType]):
         if not name:
             raise ValueError(f"Collection name must be provided: alias: {alias} metadata: {metadata}")
         collection_cls = self.collection_class
-        collection = collection_cls(name=name, alias=alias, parent=self, metadata=metadata)
-        if metadata and metadata.source_location:
+        collection = collection_cls(name=name, parent=self, metadata=metadata)
+        if alias:
+            collection.metadata.alias = alias
+        if metadata and metadata.source:
             collection.load_from_source()
         if metadata and metadata.attributes:
             sv = self.schema_view
@@ -321,7 +316,7 @@ class Database(ABC, Generic[CollectionType]):
             alias = name
         self._collections[alias] = collection
         if recreate_if_exists:
-            logger.debug(f"Recreating collection {collection.name}")
+            logger.debug(f"Recreating collection {collection.alias}")
             collection.delete_where({}, missing_ok=True)
         return collection
 
@@ -339,7 +334,7 @@ class Database(ABC, Generic[CollectionType]):
         >>> collections = db.list_collections()
         >>> len(collections)
         2
-        >>> [c.name for c in collections]
+        >>> [c.target_class_name for c in collections]
         ['Person', 'Product']
 
         :param include_internal: include internal collections
@@ -367,7 +362,7 @@ class Database(ABC, Generic[CollectionType]):
         ['Person', 'Product']
 
         """
-        return [c.name for c in self.list_collections(**kwargs)]
+        return [c.alias for c in self.list_collections(**kwargs)]
 
     def get_collection(
         self, name: str, type: Optional[str] = None, create_if_not_exists=True, **kwargs
@@ -410,7 +405,7 @@ class Database(ABC, Generic[CollectionType]):
         """
         Initialize collections.
 
-        Not typically called directly: consider making hidden
+        TODO: Not typically called directly: consider making this private
         :return:
         """
         raise NotImplementedError
@@ -502,7 +497,7 @@ class Database(ABC, Generic[CollectionType]):
         >>> sorted(collection.class_definition().slots)
         ['capital', 'code', 'continent', 'languages', 'name']
 
-        :param schema_view:
+        :param schema_view: can be either a path to the schema, or a SchemaView object
         :return:
         """
         if isinstance(schema_view, Path):
@@ -585,7 +580,15 @@ class Database(ABC, Generic[CollectionType]):
 
         :return: A schema view
         """
-        raise NotImplementedError()
+        logger.info(f"Inducing schema view for {self.handle}")
+        from linkml_runtime.utils.schema_builder import SchemaBuilder
+
+        sb = SchemaBuilder()
+
+        for collection_name in self.list_collection_names():
+            coll = self.get_collection(collection_name)
+            sb.add_class(coll.target_class_name)
+        return SchemaView(sb.schema)
 
     def iter_validate_database(self, **kwargs) -> Iterator["ValidationResult"]:
         """
@@ -683,6 +686,21 @@ class Database(ABC, Generic[CollectionType]):
         """
         Drop the database and all collections.
 
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> path = Path("/tmp/test.db")
+        >>> path.parent.mkdir(exist_ok=True, parents=True)
+        >>> db = client.attach_database(f"duckdb:///{path}")
+        >>> db.store({"persons": [{"id": "P1", "name": "John", "age_in_years": 30}]})
+        >>> coll = db.get_collection("persons")
+        >>> coll.find({}).num_rows
+        1
+        >>> db.drop()
+        >>> db = client.attach_database("duckdb:///tmp/test.db", alias="test")
+        >>> coll = db.get_collection("persons")
+        >>> coll.find({}).num_rows
+        0
+
         :param kwargs: additional arguments
         """
         raise NotImplementedError()

linkml_store/api/stores/duckdb/duckdb_collection.py

@@ -18,6 +18,9 @@ logger = logging.getLogger(__name__)
 class DuckDBCollection(Collection):
     _table_created: bool = None
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
     def insert(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         logger.debug(f"Inserting {len(objs)}")
         if not isinstance(objs, list):
@@ -123,6 +126,17 @@ class DuckDBCollection(Collection):
         t = Table(self.alias, metadata_obj, *cols)
         return t
 
+    def _check_if_initialized(self) -> bool:
+        # if self._initialized:
+        #    return True
+        query = Query(
+            from_table="information_schema.tables", where_clause={"table_type": "BASE TABLE", "table_name": self.alias}
+        )
+        qr = self.parent.query(query)
+        if qr.num_rows > 0:
+            return True
+        return False
+
     def _create_table(self, cd: ClassDefinition):
         if self._table_created or self.metadata.is_prepopulated:
             logger.info(f"Already have table for: {cd.name}")
@@ -134,6 +148,7 @@ class DuckDBCollection(Collection):
         if qr.num_rows > 0:
             logger.info(f"Table already exists for {cd.name}")
             self._table_created = True
+            self._initialized = True
             self.metadata.is_prepopulated = True
             return
         logger.info(f"Creating table for {cd.name}")
@@ -144,4 +159,5 @@ class DuckDBCollection(Collection):
             conn.execute(text(ddl))
             conn.commit()
         self._table_created = True
+        self._initialized = True
         self.metadata.is_prepopulated = True