linkml-store 0.1.10__py3-none-any.whl → 0.1.12__py3-none-any.whl

linkml_store/api/database.py

@@ -19,7 +19,7 @@ from typing import (
 )
 
 from linkml_store.api.types import CollectionType
-from linkml_store.utils.format_utils import load_objects, render_output
+from linkml_store.utils.format_utils import Format, load_objects, render_output
 from linkml_store.utils.patch_utils import PatchDict
 
 try:
@@ -149,26 +149,19 @@ class Database(ABC, Generic[CollectionType]):
     def _initialize_collections(self):
         if not self.metadata.collections:
             return
-        for name, collection_config in self.metadata.collections.items():
-            alias = collection_config.alias
-            typ = collection_config.type
-            # if typ and alias is None:
-            #    alias = name
-            # if typ is None:
-            #    typ = name
-            # collection = self.create_collection(
-            #    typ, alias=alias, metadata=collection_config.metadata
-            # )
-            if False and typ is not None:
-                if not alias:
-                    alias = name
-                name = typ
-            if not collection_config.name:
-                collection_config.name = name
-            _collection = self.create_collection(name, alias=alias, metadata=collection_config)
+        for k, collection_config in self.metadata.collections.items():
+            if collection_config.alias:
+                if collection_config.alias != k:
+                    raise ValueError(f"Alias mismatch: {collection_config.alias} != {k}")
+            alias = k
+            typ = collection_config.type or alias
+            _collection = self.create_collection(typ, alias=alias, metadata=collection_config)
+            assert _collection.alias == alias
+            assert _collection.target_class_name == typ
             if collection_config.attributes:
+                # initialize schema
                 sv = self.schema_view
-                cd = ClassDefinition(name, attributes=collection_config.attributes)
+                cd = ClassDefinition(typ, attributes=collection_config.attributes)
                 sv.schema.classes[cd.name] = cd
                 sv.set_modified()
                 # assert collection.class_definition() is not None
@@ -275,7 +268,7 @@ class Database(ABC, Generic[CollectionType]):
         metadata: Optional[CollectionConfig] = None,
         recreate_if_exists=False,
         **kwargs,
-    ) -> Collection:
+    ) -> CollectionType:
         """
         Create a new collection in the current database.
 
@@ -307,8 +300,10 @@ class Database(ABC, Generic[CollectionType]):
         if not name:
             raise ValueError(f"Collection name must be provided: alias: {alias} metadata: {metadata}")
         collection_cls = self.collection_class
-        collection = collection_cls(name=name, alias=alias, parent=self, metadata=metadata)
-        if metadata and metadata.source_location:
+        collection = collection_cls(name=name, parent=self, metadata=metadata)
+        if alias:
+            collection.metadata.alias = alias
+        if metadata and metadata.source:
             collection.load_from_source()
         if metadata and metadata.attributes:
             sv = self.schema_view
@@ -321,7 +316,7 @@ class Database(ABC, Generic[CollectionType]):
             alias = name
         self._collections[alias] = collection
         if recreate_if_exists:
-            logger.debug(f"Recreating collection {collection.name}")
+            logger.debug(f"Recreating collection {collection.alias}")
             collection.delete_where({}, missing_ok=True)
         return collection
 
@@ -339,7 +334,7 @@ class Database(ABC, Generic[CollectionType]):
         >>> collections = db.list_collections()
         >>> len(collections)
         2
-        >>> [c.name for c in collections]
+        >>> [c.target_class_name for c in collections]
         ['Person', 'Product']
 
         :param include_internal: include internal collections
@@ -367,7 +362,7 @@ class Database(ABC, Generic[CollectionType]):
         ['Person', 'Product']
 
         """
-        return [c.name for c in self.list_collections(**kwargs)]
+        return [c.alias for c in self.list_collections(**kwargs)]
 
     def get_collection(
         self, name: str, type: Optional[str] = None, create_if_not_exists=True, **kwargs
@@ -410,7 +405,7 @@ class Database(ABC, Generic[CollectionType]):
         """
         Initialize collections.
 
-        Not typically called directly: consider making hidden
+        TODO: Not typically called directly: consider making this private
         :return:
         """
         raise NotImplementedError
@@ -502,7 +497,7 @@ class Database(ABC, Generic[CollectionType]):
         >>> sorted(collection.class_definition().slots)
         ['capital', 'code', 'continent', 'languages', 'name']
 
-        :param schema_view:
+        :param schema_view: can be either a path to the schema, or a SchemaView object
         :return:
         """
         if isinstance(schema_view, Path):
@@ -585,7 +580,15 @@ class Database(ABC, Generic[CollectionType]):
 
         :return: A schema view
         """
-        raise NotImplementedError()
+        logger.info(f"Inducing schema view for {self.handle}")
+        from linkml_runtime.utils.schema_builder import SchemaBuilder
+
+        sb = SchemaBuilder()
+
+        for collection_name in self.list_collection_names():
+            coll = self.get_collection(collection_name)
+            sb.add_class(coll.target_class_name)
+        return SchemaView(sb.schema)
 
     def iter_validate_database(self, **kwargs) -> Iterator["ValidationResult"]:
         """
@@ -683,11 +686,26 @@ class Database(ABC, Generic[CollectionType]):
         """
         Drop the database and all collections.
 
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> path = Path("/tmp/test.db")
+        >>> path.parent.mkdir(exist_ok=True, parents=True)
+        >>> db = client.attach_database(f"duckdb:///{path}")
+        >>> db.store({"persons": [{"id": "P1", "name": "John", "age_in_years": 30}]})
+        >>> coll = db.get_collection("persons")
+        >>> coll.find({}).num_rows
+        1
+        >>> db.drop()
+        >>> db = client.attach_database("duckdb:///tmp/test.db", alias="test")
+        >>> coll = db.get_collection("persons")
+        >>> coll.find({}).num_rows
+        0
+
         :param kwargs: additional arguments
         """
         raise NotImplementedError()
 
-    def import_database(self, location: str, source_format: Optional[str] = None, **kwargs):
+    def import_database(self, location: str, source_format: Optional[Union[str, Format]] = None, **kwargs):
         """
         Import a database from a file or location.
 
@@ -695,11 +713,27 @@ class Database(ABC, Generic[CollectionType]):
         :param source_format: source format
         :param kwargs: additional arguments
         """
+        if isinstance(source_format, str):
+            source_format = Format(source_format)
+        if isinstance(source_format, Format):
+            if source_format.is_dump_format() and source_format in [Format.SQLDUMP_DUCKDB, Format.DUMP_MONGODB]:
+                # import into a test instance
+                tmp_handle = source_format.value
+                client = self.parent
+                tmp_db = client.attach_database(tmp_handle, alias="tmp")
+                # TODO: check for infinite recursion
+                tmp_db.import_database(location, source_format=source_format)
+                obj = {}
+                for coll in tmp_db.list_collections():
+                    qr = coll.find({}, limit=-1)
+                    obj[coll.alias] = qr.rows
+                self.store(obj)
+                return
         objects = load_objects(location, format=source_format)
         for obj in objects:
             self.store(obj)
 
-    def export_database(self, location: str, target_format: Optional[str] = None, **kwargs):
+    def export_database(self, location: str, target_format: Optional[Union[str, Format]] = None, **kwargs):
         """
         Export a database to a file or location.
 
@@ -708,10 +742,23 @@ class Database(ABC, Generic[CollectionType]):
         :param kwargs: additional arguments
         """
         obj = {}
+        if isinstance(target_format, str):
+            target_format = Format(target_format)
         for coll in self.list_collections():
             qr = coll.find({}, limit=-1)
             obj[coll.alias] = qr.rows
         logger.info(f"Exporting object with {len(obj)} collections to {location} in {target_format} format")
+        if isinstance(target_format, Format):
+            if target_format.is_dump_format() and target_format in [Format.SQLDUMP_DUCKDB, Format.DUMP_MONGODB]:
+                tmp_handle = target_format.value
+                client = self.parent
+                tmp_db = client.attach_database(tmp_handle, alias="tmp")
+                tmp_db.store(obj)
+                # TODO: check for infinite recursion
+                tmp_db.export_database(location, target_format=target_format)
+                return
+        if Path(location).is_dir():
+            raise ValueError(f"{location} is a directory; cannot write {target_format} to a dir")
         with open(location, "w", encoding="utf-8") as stream:
             stream.write(render_output(obj, format=target_format))
 

linkml_store/api/stores/duckdb/duckdb_collection.py

@@ -18,6 +18,9 @@ logger = logging.getLogger(__name__)
 class DuckDBCollection(Collection):
     _table_created: bool = None
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
     def insert(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         logger.debug(f"Inserting {len(objs)}")
         if not isinstance(objs, list):
@@ -123,6 +126,17 @@ class DuckDBCollection(Collection):
         t = Table(self.alias, metadata_obj, *cols)
         return t
 
+    def _check_if_initialized(self) -> bool:
+        # if self._initialized:
+        #    return True
+        query = Query(
+            from_table="information_schema.tables", where_clause={"table_type": "BASE TABLE", "table_name": self.alias}
+        )
+        qr = self.parent.query(query)
+        if qr.num_rows > 0:
+            return True
+        return False
+
     def _create_table(self, cd: ClassDefinition):
         if self._table_created or self.metadata.is_prepopulated:
             logger.info(f"Already have table for: {cd.name}")
@@ -134,6 +148,7 @@ class DuckDBCollection(Collection):
         if qr.num_rows > 0:
             logger.info(f"Table already exists for {cd.name}")
             self._table_created = True
+            self._initialized = True
             self.metadata.is_prepopulated = True
             return
         logger.info(f"Creating table for {cd.name}")
@@ -144,4 +159,5 @@ class DuckDBCollection(Collection):
             conn.execute(text(ddl))
             conn.commit()
         self._table_created = True
+        self._initialized = True
         self.metadata.is_prepopulated = True

linkml_store/api/stores/duckdb/duckdb_database.py

@@ -1,11 +1,10 @@
 import json
 import logging
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Union
 
 import pandas as pd
 import sqlalchemy
-from duckdb import DuckDBPyConnection
 from linkml_runtime import SchemaView
 from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition
 from linkml_runtime.utils.schema_builder import SchemaBuilder
@@ -14,6 +13,7 @@ from sqlalchemy import NullPool, text
 from linkml_store.api import Database
 from linkml_store.api.queries import Query, QueryResult
 from linkml_store.api.stores.duckdb.duckdb_collection import DuckDBCollection
+from linkml_store.utils.format_utils import Format
 from linkml_store.utils.sql_utils import introspect_schema, query_to_sql
 
 TYPE_MAP = {
@@ -26,6 +26,8 @@ TYPE_MAP = {
     "JSON": "Any",
 }
 
+MEMORY_HANDLE = "duckdb:///:memory:"
+
 
 logger = logging.getLogger(__name__)
 
@@ -43,13 +45,13 @@ class DuckDBDatabase(Database):
     types are used for nested inlined objects.
     """
 
-    _connection: DuckDBPyConnection = None
+    # _connection: DuckDBPyConnection = None
     _engine: sqlalchemy.Engine = None
     collection_class = DuckDBCollection
 
     def __init__(self, handle: Optional[str] = None, recreate_if_exists: bool = False, **kwargs):
         if handle is None:
-            handle = "duckdb:///:memory:"
+            handle = MEMORY_HANDLE
         if recreate_if_exists:
             path = Path(handle.replace("duckdb:///", ""))
             if path.exists():
@@ -76,6 +78,17 @@ class DuckDBDatabase(Database):
     def close(self, **kwargs):
         self.engine.dispose()
 
+    def drop(self, missing_ok=True, **kwargs):
+        self.close()
+        if self.handle == MEMORY_HANDLE:
+            return
+        path = Path(self.handle.replace("duckdb:///", ""))
+        if path.exists():
+            path.unlink()
+        else:
+            if not missing_ok:
+                raise FileNotFoundError(f"Database file not found: {path}")
+
     def query(self, query: Query, **kwargs) -> QueryResult:
         json_encoded_cols = []
         if query.from_table:
@@ -94,7 +107,8 @@ class DuckDBDatabase(Database):
             if sv:
                 cd = None
                 for c in self._collections.values():
-                    if c.name == query.from_table or c.metadata.alias == query.from_table:
+                    # if c.name == query.from_table or c.metadata.alias == query.from_table:
+                    if c.alias == query.from_table or c.target_class_name == query.from_table:
                         cd = c.class_definition()
                         break
                 if cd:
@@ -188,3 +202,31 @@ class DuckDBDatabase(Database):
                     cls = ClassDefinition(name=collection_metadata.type, attributes=collection_metadata.attributes)
                     schema.classes[cls.name] = cls
         return SchemaView(schema)
+
+    def export_database(self, location: str, target_format: Optional[Union[str, Format]] = None, **kwargs):
+        if target_format == "duckdb" or target_format == Format.SQLDUMP_DUCKDB:
+            path = Path(location)
+            if path.exists():
+                if path.is_file():
+                    path.unlink()
+            with self.engine.connect() as conn:
+                sql = text(f"EXPORT DATABASE '{location}'")
+                conn.execute(sql)
+        else:
+            super().export_database(location, target_format=target_format, **kwargs)
+
+    def import_database(self, location: str, source_format: Optional[str] = None, **kwargs):
+        """
+        Import a database from a file or location.
+
+        :param location: location of the file
+        :param source_format: source format
+        :param kwargs: additional arguments
+        """
+        if source_format == Format.SQLDUMP_DUCKDB.value or source_format == Format.SQLDUMP_DUCKDB:
+            with self.engine.connect() as conn:
+                sql = text(f"IMPORT DATABASE '{location}'")
+                conn.execute(sql)
+                conn.commit()
+        else:
+            super().import_database(location, source_format=source_format, **kwargs)

linkml_store/api/stores/filesystem/filesystem_collection.py

@@ -31,7 +31,7 @@ class FileSystemCollection(Collection[DatabaseType]):
 
     @property
     def path_to_file(self):
-        return Path(self.parent.directory_path) / f"{self.name}.{self.file_format}"
+        return Path(self.parent.directory_path) / f"{self.alias}.{self.file_format}"
 
     @property
     def objects_as_list(self) -> List[OBJECT]:
@@ -150,13 +150,20 @@ class FileSystemCollection(Collection[DatabaseType]):
         curr_objects = [o for o in self.objects_as_list if not matches(o)]
         self._set_objects(curr_objects)
 
-    def query(self, query: Query, **kwargs) -> QueryResult:
-
+    def query(self, query: Query, limit: Optional[int] = None, offset: Optional[int] = None, **kwargs) -> QueryResult:
+        limit = limit or query.limit
+        offset = offset or query.offset
+        if offset is None:
+            offset = 0
         where = query.where_clause or {}
         match = mongo_query_to_match_function(where)
         rows = [o for o in self.objects_as_list if match(o)]
         count = len(rows)
-        return QueryResult(query=query, num_rows=count, rows=rows)
+        if limit is None or limit < 0:
+            limit = count
+        # TODO: avoid recalculating
+        returned_row = rows[offset : offset + limit]
+        return QueryResult(query=query, num_rows=count, rows=returned_row)
 
     def query_facets(
         self, where: Dict = None, facet_columns: List[str] = None, facet_limit=DEFAULT_FACET_LIMIT, **kwargs

linkml_store/api/stores/filesystem/filesystem_database.py

@@ -9,6 +9,7 @@ from linkml_runtime import SchemaView
 from linkml_store.api import Database
 from linkml_store.api.config import DatabaseConfig
 from linkml_store.api.stores.filesystem.filesystem_collection import FileSystemCollection
+from linkml_store.utils.file_utils import safe_remove_directory
 from linkml_store.utils.format_utils import Format, load_objects
 
 logger = logging.getLogger(__name__)
@@ -20,6 +21,8 @@ class FileSystemDatabase(Database):
     directory_path: Optional[Path] = None
     default_file_format: Optional[str] = None
 
+    no_backup_on_drop: bool = False
+
     def __init__(self, handle: Optional[str] = None, **kwargs):
         handle = handle.replace("file:", "")
         if handle.startswith("//"):
@@ -43,6 +46,12 @@ class FileSystemDatabase(Database):
     def close(self, **kwargs):
         pass
 
+    def drop(self, no_backup=False, **kwargs):
+        self.close()
+        path = self.directory_path
+        if path.exists():
+            safe_remove_directory(path, no_backup=self.no_backup_on_drop or no_backup)
+
     def init_collections(self):
         metadata = self.metadata
         if self._collections is None:
@@ -63,7 +72,7 @@ class FileSystemDatabase(Database):
                     self._collections[n] = collection
                     collection._set_objects(objs)
 
-    def induce_schema_view(self) -> SchemaView:
+    def xxxinduce_schema_view(self) -> SchemaView:
         logger.info(f"Inducing schema view for {self.handle}")
         sb = SchemaBuilder()
 

linkml_store/api/stores/mongodb/mongodb_collection.py

@@ -23,11 +23,15 @@ class MongoDBCollection(Collection):
 
     @property
     def mongo_collection(self) -> MongoCollection:
-        if not self.name:
+        # collection_name = self.alias or self.name
+        collection_name = self.alias
+        if not collection_name:
             raise ValueError("Collection name not set")
-        collection_name = self.alias or self.name
         return self.parent.native_db[collection_name]
 
+    def _check_if_initialized(self) -> bool:
+        return self.alias in self.parent.native_db.list_collection_names()
+
     def insert(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         if not isinstance(objs, list):
             objs = [objs]

linkml_store/api/stores/mongodb/mongodb_database.py

@@ -1,17 +1,18 @@
 # mongodb_database.py
 
 import logging
-from typing import Optional
+from pathlib import Path
+from typing import Optional, Union
 
-from linkml_runtime import SchemaView
-from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition
-from linkml_runtime.utils.schema_builder import SchemaBuilder
 from pymongo import MongoClient
 from pymongo.database import Database as NativeDatabase
 
 from linkml_store.api import Database
 from linkml_store.api.queries import Query, QueryResult
 from linkml_store.api.stores.mongodb.mongodb_collection import MongoDBCollection
+from linkml_store.utils.file_utils import safe_remove_directory
+from linkml_store.utils.format_utils import Format
+from linkml_store.utils.mongodb_utils import import_mongodb
 
 logger = logging.getLogger(__name__)
 
@@ -30,6 +31,8 @@ class MongoDBDatabase(Database):
     def __init__(self, handle: Optional[str] = None, **kwargs):
         if handle is None:
             handle = "mongodb://localhost:27017/test"
+        if handle == "mongodb":
+            handle = "mongodb://localhost:27017/temporary"
         super().__init__(handle=handle, **kwargs)
 
     @property
@@ -63,10 +66,9 @@ class MongoDBDatabase(Database):
             self._native_client.close()
 
     def drop(self, **kwargs):
-        self.native_client.drop_database(self.metadata.alias)
+        self.native_client.drop_database(self.native_db.name)
 
     def query(self, query: Query, **kwargs) -> QueryResult:
-        # TODO: DRY
         if query.from_table:
             collection = self.get_collection(query.from_table)
             return collection.query(query, **kwargs)
@@ -82,33 +84,26 @@ class MongoDBDatabase(Database):
                 collection = MongoDBCollection(name=collection_name, parent=self)
                 self._collections[collection_name] = collection
 
-    def induce_schema_view(self) -> SchemaView:
-        logger.info(f"Inducing schema view for {self.handle}")
-        sb = SchemaBuilder()
-        schema = sb.schema
+    def export_database(self, location: str, target_format: Optional[Union[str, Format]] = None, **kwargs):
+        if target_format == Format.DUMP_MONGODB.value or target_format == Format.DUMP_MONGODB:
+            path = Path(location)
+            if path.exists():
+                safe_remove_directory(path, no_backup=True)
+            from linkml_store.utils.mongodb_utils import export_mongodb
 
-        for collection_name in self.native_db.list_collection_names():
-            sb.add_class(collection_name)
-            mongo_collection = self.native_db[collection_name]
-            sample_doc = mongo_collection.find_one()
-            if sample_doc:
-                for field, value in sample_doc.items():
-                    if field == "_id":
-                        continue
-                    sd = SlotDefinition(field)
-                    if isinstance(value, list):
-                        sd.multivalued = True
-                    if isinstance(value, dict):
-                        sd.inlined = True
-                    sb.schema.classes[collection_name].attributes[sd.name] = sd
-
-        sb.add_defaults()
-        for cls_name in schema.classes:
-            if cls_name in self.metadata.collections:
-                collection_metadata = self.metadata.collections[cls_name]
-                if collection_metadata.attributes:
-                    del schema.classes[cls_name]
-                    cls = ClassDefinition(name=collection_metadata.type, attributes=collection_metadata.attributes)
-                    schema.classes[cls.name] = cls
-
-        return SchemaView(schema)
+            export_mongodb(self.handle, location)
+        else:
+            super().export_database(location, target_format=target_format, **kwargs)
+
+    def import_database(self, location: str, source_format: Optional[str] = None, **kwargs):
+        """
+        Import a database from a file or location.
+
+        :param location: location of the file
+        :param source_format: source format
+        :param kwargs: additional arguments
+        """
+        if source_format == Format.DUMP_MONGODB.value or source_format == Format.DUMP_MONGODB:
+            import_mongodb(self.handle, location, drop=True)
+        else:
+            super().import_database(location, source_format=source_format, **kwargs)

linkml_store/api/stores/solr/solr_collection.py

@@ -18,7 +18,7 @@ class SolrCollection(Collection):
     @property
     def _collection_base(self) -> str:
         if self.parent.use_cores:
-            base_url = f"{self.parent.base_url}/{self.name}"
+            base_url = f"{self.parent.base_url}/{self.alias}"
         else:
             base_url = self.parent.base_url
         return base_url
@@ -37,7 +37,7 @@ class SolrCollection(Collection):
         if not qfs:
             raise ValueError("No searchable slots configured for Solr collection")
         solr_query = self._build_solr_query(where, search_term=query, extra={"defType": index_name, "qf": qfs})
-        logger.info(f"Querying Solr collection {self.name} with query: {solr_query}")
+        logger.info(f"Querying Solr collection {self.alias} with query: {solr_query}")
 
         response = requests.get(f"{self._collection_base}/select", params=solr_query)
         response.raise_for_status()
@@ -50,7 +50,7 @@ class SolrCollection(Collection):
 
     def query(self, query: Query, **kwargs) -> QueryResult:
         solr_query = self._build_solr_query(query)
-        logger.info(f"Querying Solr collection {self.name} with query: {solr_query}")
+        logger.info(f"Querying Solr collection {self.alias} with query: {solr_query}")
 
         response = requests.get(f"{self._collection_base}/select", params=solr_query)
         response.raise_for_status()
@@ -69,7 +69,7 @@ class SolrCollection(Collection):
         solr_query["facet.field"] = facet_columns
         solr_query["facet.limit"] = facet_limit
 
-        logger.info(f"Querying Solr collection {self.name} for facets with query: {solr_query}")
+        logger.info(f"Querying Solr collection {self.alias} for facets with query: {solr_query}")
 
         response = requests.get(f"{self._collection_base}/select", params=solr_query)
         response.raise_for_status()