linkml-store 0.1.11__tar.gz → 0.1.13__tar.gz

{linkml_store-0.1.11 → linkml_store-0.1.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: linkml-store
-Version: 0.1.11
+Version: 0.1.13
 Summary: linkml-store
 License: MIT
 Author: Author 1
@@ -21,6 +21,7 @@ Provides-Extra: h5py
 Provides-Extra: llm
 Provides-Extra: map
 Provides-Extra: mongodb
+Provides-Extra: neo4j
 Provides-Extra: pyarrow
 Provides-Extra: renderer
 Provides-Extra: tests
@@ -28,7 +29,7 @@ Provides-Extra: validation
 Requires-Dist: black (>=24.0.0) ; extra == "tests"
 Requires-Dist: chromadb ; extra == "chromadb"
 Requires-Dist: click
-Requires-Dist: duckdb (>=0.10.1,<0.11.0)
+Requires-Dist: duckdb (>=0.10.1)
 Requires-Dist: duckdb-engine (>=0.11.2)
 Requires-Dist: fastapi ; extra == "fastapi"
 Requires-Dist: frictionless ; extra == "frictionless"
@@ -41,8 +42,11 @@ Requires-Dist: linkml_map ; extra == "map"
 Requires-Dist: linkml_renderer ; extra == "renderer"
 Requires-Dist: llm ; extra == "llm"
 Requires-Dist: matplotlib ; extra == "analytics"
+Requires-Dist: neo4j ; extra == "neo4j"
+Requires-Dist: networkx ; extra == "neo4j"
 Requires-Dist: pandas (>=2.2.1) ; extra == "analytics"
 Requires-Dist: plotly ; extra == "analytics"
+Requires-Dist: py2neo ; extra == "neo4j"
 Requires-Dist: pyarrow ; extra == "pyarrow"
 Requires-Dist: pydantic (>=2.0.0,<3.0.0)
 Requires-Dist: pymongo ; extra == "mongodb"

{linkml_store-0.1.11 → linkml_store-0.1.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "linkml-store"
-version = "0.1.11"
+version = "0.1.13"
 description = "linkml-store"
 authors = ["Author 1 <author@org.org>"]
 license = "MIT"
@@ -13,7 +13,7 @@ pydantic = "^2.0.0"
 linkml-runtime = ">=1.8.0"
 streamlit = { version = "^1.32.2", optional = true }
 sqlalchemy = "*"
-duckdb = "^0.10.1"
+duckdb = ">=0.10.1"
 duckdb-engine = ">=0.11.2"
 matplotlib = { version = "*", optional = true }
 seaborn = { version = "*", optional = true }
@@ -22,6 +22,9 @@ pystow = "^0.5.4"
 black = { version=">=24.0.0", optional = true }
 llm = { version="*", optional = true }
 pymongo = { version="*", optional = true }
+neo4j = { version="*", optional = true }
+py2neo = { version="*", optional = true }
+networkx = { version="*", optional = true }
 chromadb = { version="*", optional = true }
 pyarrow = { version="*", optional = true }
 h5py = { version="*", optional = true }
@@ -66,6 +69,7 @@ app = ["streamlit"]
 tests = ["black"]
 llm = ["llm"]
 mongodb = ["pymongo"]
+neo4j = ["neo4j", "py2neo", "networkx"]
 chromadb = ["chromadb"]
 h5py = ["h5py"]
 pyarrow = ["pyarrow"]

{linkml_store-0.1.11 → linkml_store-0.1.13}/src/linkml_store/api/client.py

@@ -11,6 +11,7 @@ from linkml_store.api.stores.chromadb.chromadb_database import ChromaDBDatabase
 from linkml_store.api.stores.duckdb.duckdb_database import DuckDBDatabase
 from linkml_store.api.stores.filesystem.filesystem_database import FileSystemDatabase
 from linkml_store.api.stores.mongodb.mongodb_database import MongoDBDatabase
+from linkml_store.api.stores.neo4j.neo4j_database import Neo4jDatabase
 from linkml_store.api.stores.solr.solr_database import SolrDatabase
 
 logger = logging.getLogger(__name__)
@@ -21,6 +22,7 @@ HANDLE_MAP = {
     "solr": SolrDatabase,
     "mongodb": MongoDBDatabase,
     "chromadb": ChromaDBDatabase,
+    "neo4j": Neo4jDatabase,
     "file": FileSystemDatabase,
 }
 

{linkml_store-0.1.11 → linkml_store-0.1.13}/src/linkml_store/api/collection.py

@@ -4,7 +4,21 @@ import hashlib
 import logging
 from collections import defaultdict
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, ClassVar, Dict, Generic, Iterator, List, Optional, TextIO, Tuple, Type, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    ClassVar,
+    Dict,
+    Generic,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    TextIO,
+    Tuple,
+    Type,
+    Union,
+)
 
 import numpy as np
 from linkml_runtime import SchemaView
@@ -202,6 +216,12 @@ class Collection(Generic[DatabaseType]):
             self._materialize_derivations()
             self._initialized = True
 
+    def _pre_insert_hook(self, objs: List[OBJECT], **kwargs):
+        if self.metadata.validate_modifications:
+            errors = list(self.iter_validate_collection(objs))
+            if errors:
+                raise ValueError(f"Validation errors: {errors}")
+
     def _post_insert_hook(self, objs: List[OBJECT], **kwargs):
         self._initialized = True
         patches = [{"op": "add", "path": "/0", "value": obj} for obj in objs]
@@ -346,7 +366,10 @@ class Collection(Generic[DatabaseType]):
         id_field = self.identifier_attribute_name
         if not id_field:
             raise ValueError(f"No identifier for {self.name}")
-        return self.find({id_field: ids})
+        if len(ids) == 1:
+            return self.find({id_field: ids[0]})
+        else:
+            return self.find({id_field: {"$in": ids}})
 
     def get_one(self, id: IDENTIFIER, **kwargs) -> Optional[OBJECT]:
         """
@@ -518,7 +541,7 @@ class Collection(Generic[DatabaseType]):
         :return:
         """
         cd = self.class_definition()
-        return cd is not None
+        return cd is not None and cd.attributes
 
     def load_from_source(self, load_if_exists=False):
         """
@@ -535,11 +558,19 @@ class Collection(Generic[DatabaseType]):
             kwargs = source.arguments or {}
             if source.local_path:
                 objects = load_objects(
-                    metadata.source.local_path, format=source.format, expected_type=source.expected_type, **kwargs
+                    metadata.source.local_path,
+                    format=source.format,
+                    expected_type=source.expected_type,
+                    compression=source.compression,
+                    **kwargs,
                 )
             elif metadata.source.url:
                 objects = load_objects_from_url(
-                    metadata.source.url, format=source.format, expected_type=source.expected_type, **kwargs
+                    metadata.source.url,
+                    format=source.format,
+                    expected_type=source.expected_type,
+                    compression=source.compression,
+                    **kwargs,
                 )
         self.insert(objects)
 
@@ -746,6 +777,7 @@ class Collection(Generic[DatabaseType]):
         sv: SchemaView = self.parent.schema_view
         if sv:
             cls = sv.get_class(self.target_class_name)
+            # cls = sv.schema.classes[self.target_class_name]
             if cls and not cls.attributes:
                 if not sv.class_induced_slots(cls.name):
                     for att in self._induce_attributes():
@@ -868,7 +900,7 @@ class Collection(Generic[DatabaseType]):
                     exact_dimensions_list.append(v.shape)
                     break
                 if isinstance(v, list):
-                    v = v[0]
+                    v = v[0] if v else None
                     multivalueds.append(True)
                 elif isinstance(v, dict):
                     v = list(v.values())[0]
@@ -966,11 +998,14 @@ class Collection(Generic[DatabaseType]):
         patches_from_objects_lists(src_objs, tgt_objs, primary_key=primary_key)
         return patches_from_objects_lists(src_objs, tgt_objs, primary_key=primary_key)
 
-    def iter_validate_collection(self, **kwargs) -> Iterator["ValidationResult"]:
+    def iter_validate_collection(
+        self, objects: Optional[Iterable[OBJECT]] = None, **kwargs
+    ) -> Iterator["ValidationResult"]:
         """
         Validate the contents of the collection
 
         :param kwargs:
+        :param objects: objects to validate
         :return: iterator over validation results
         """
         from linkml.validator import JsonschemaValidationPlugin, Validator
@@ -980,10 +1015,24 @@ class Collection(Generic[DatabaseType]):
         cd = self.class_definition()
         if not cd:
             raise ValueError(f"Cannot find class definition for {self.target_class_name}")
+        type_designator = None
+        for att in self.parent.schema_view.class_induced_slots(cd.name):
+            if att.designates_type:
+                type_designator = att.name
         class_name = cd.name
-        for obj in self.find_iter(**kwargs):
+        if objects is None:
+            objects = self.find_iter(**kwargs)
+        for obj in objects:
             obj = clean_empties(obj)
-            yield from validator.iter_results(obj, class_name)
+            v_class_name = class_name
+            if type_designator is not None:
+                # TODO: move type designator logic to core linkml
+                this_class_name = obj.get(type_designator)
+                if this_class_name:
+                    if ":" in this_class_name:
+                        this_class_name = this_class_name.split(":")[-1]
+                    v_class_name = this_class_name
+            yield from validator.iter_results(obj, v_class_name)
 
     def commit(self):
         """

{linkml_store-0.1.11 → linkml_store-0.1.13}/src/linkml_store/api/config.py

@@ -2,6 +2,8 @@ from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel, Field
 
+from linkml_store.graphs.graph_map import GraphProjection
+
 
 class ConfiguredBaseModel(BaseModel, extra="forbid"):
     """
@@ -33,6 +35,7 @@ class CollectionSource(ConfiguredBaseModel):
     refresh_interval_days: Optional[float] = None
     expected_type: Optional[str] = None
     format: Optional[str] = None
+    compression: Optional[str] = None
     arguments: Optional[Dict[str, Any]] = None
 
 
@@ -73,11 +76,19 @@ class CollectionConfig(ConfiguredBaseModel):
         default=None,
         description="Metadata about the source",
     )
-    # TODO: derived_from
     derived_from: Optional[List[DerivationConfiguration]] = Field(
         default=None,
         description="LinkML-Map derivations",
     )
+    page_size: Optional[int] = Field(default=None, description="Suggested page size (items per page) in apps and APIs")
+    graph_projection: Optional[GraphProjection] = Field(
+        default=None,
+        description="Optional graph projection configuration",
+    )
+    validate_modifications: Optional[bool] = Field(
+        default=False,
+        description="Whether to validate inserts, updates, and deletes",
+    )
 
 
 class DatabaseConfig(ConfiguredBaseModel):

{linkml_store-0.1.11 → linkml_store-0.1.13}/src/linkml_store/api/database.py

@@ -19,7 +19,7 @@ from typing import (
 )
 
 from linkml_store.api.types import CollectionType
-from linkml_store.utils.format_utils import load_objects, render_output
+from linkml_store.utils.format_utils import Format, load_objects, render_output
 from linkml_store.utils.patch_utils import PatchDict
 
 try:
@@ -505,8 +505,10 @@ class Database(ABC, Generic[CollectionType]):
         if isinstance(schema_view, str):
             schema_view = SchemaView(schema_view)
         self._schema_view = schema_view
+        # self._schema_view = SchemaView(schema_view.materialize_derived_schema())
         if not self._collections:
             return
+
         # align with induced schema
         roots = [c for c in schema_view.all_classes().values() if c.tree_root]
         if len(roots) == 0:
@@ -705,7 +707,7 @@ class Database(ABC, Generic[CollectionType]):
         """
         raise NotImplementedError()
 
-    def import_database(self, location: str, source_format: Optional[str] = None, **kwargs):
+    def import_database(self, location: str, source_format: Optional[Union[str, Format]] = None, **kwargs):
         """
         Import a database from a file or location.
 
@@ -713,11 +715,27 @@ class Database(ABC, Generic[CollectionType]):
         :param source_format: source format
         :param kwargs: additional arguments
         """
+        if isinstance(source_format, str):
+            source_format = Format(source_format)
+        if isinstance(source_format, Format):
+            if source_format.is_dump_format() and source_format in [Format.SQLDUMP_DUCKDB, Format.DUMP_MONGODB]:
+                # import into a test instance
+                tmp_handle = source_format.value
+                client = self.parent
+                tmp_db = client.attach_database(tmp_handle, alias="tmp")
+                # TODO: check for infinite recursion
+                tmp_db.import_database(location, source_format=source_format)
+                obj = {}
+                for coll in tmp_db.list_collections():
+                    qr = coll.find({}, limit=-1)
+                    obj[coll.alias] = qr.rows
+                self.store(obj)
+                return
         objects = load_objects(location, format=source_format)
         for obj in objects:
             self.store(obj)
 
-    def export_database(self, location: str, target_format: Optional[str] = None, **kwargs):
+    def export_database(self, location: str, target_format: Optional[Union[str, Format]] = None, **kwargs):
         """
         Export a database to a file or location.
 
@@ -726,10 +744,23 @@ class Database(ABC, Generic[CollectionType]):
         :param kwargs: additional arguments
         """
         obj = {}
+        if isinstance(target_format, str):
+            target_format = Format(target_format)
         for coll in self.list_collections():
             qr = coll.find({}, limit=-1)
             obj[coll.alias] = qr.rows
         logger.info(f"Exporting object with {len(obj)} collections to {location} in {target_format} format")
+        if isinstance(target_format, Format):
+            if target_format.is_dump_format() and target_format in [Format.SQLDUMP_DUCKDB, Format.DUMP_MONGODB]:
+                tmp_handle = target_format.value
+                client = self.parent
+                tmp_db = client.attach_database(tmp_handle, alias="tmp")
+                tmp_db.store(obj)
+                # TODO: check for infinite recursion
+                tmp_db.export_database(location, target_format=target_format)
+                return
+        if Path(location).is_dir():
+            raise ValueError(f"{location} is a directory; cannot write {target_format} to a dir")
         with open(location, "w", encoding="utf-8") as stream:
             stream.write(render_output(obj, format=target_format))
 

{linkml_store-0.1.11 → linkml_store-0.1.13}/src/linkml_store/api/stores/duckdb/duckdb_database.py

@@ -1,11 +1,10 @@
 import json
 import logging
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Union
 
 import pandas as pd
 import sqlalchemy
-from duckdb import DuckDBPyConnection
 from linkml_runtime import SchemaView
 from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition
 from linkml_runtime.utils.schema_builder import SchemaBuilder
@@ -14,6 +13,7 @@ from sqlalchemy import NullPool, text
 from linkml_store.api import Database
 from linkml_store.api.queries import Query, QueryResult
 from linkml_store.api.stores.duckdb.duckdb_collection import DuckDBCollection
+from linkml_store.utils.format_utils import Format
 from linkml_store.utils.sql_utils import introspect_schema, query_to_sql
 
 TYPE_MAP = {
@@ -45,7 +45,7 @@ class DuckDBDatabase(Database):
     types are used for nested inlined objects.
     """
 
-    _connection: DuckDBPyConnection = None
+    # _connection: DuckDBPyConnection = None
     _engine: sqlalchemy.Engine = None
     collection_class = DuckDBCollection
 
@@ -202,3 +202,31 @@ class DuckDBDatabase(Database):
                     cls = ClassDefinition(name=collection_metadata.type, attributes=collection_metadata.attributes)
                     schema.classes[cls.name] = cls
         return SchemaView(schema)
+
+    def export_database(self, location: str, target_format: Optional[Union[str, Format]] = None, **kwargs):
+        if target_format == "duckdb" or target_format == Format.SQLDUMP_DUCKDB:
+            path = Path(location)
+            if path.exists():
+                if path.is_file():
+                    path.unlink()
+            with self.engine.connect() as conn:
+                sql = text(f"EXPORT DATABASE '{location}'")
+                conn.execute(sql)
+        else:
+            super().export_database(location, target_format=target_format, **kwargs)
+
+    def import_database(self, location: str, source_format: Optional[str] = None, **kwargs):
+        """
+        Import a database from a file or location.
+
+        :param location: location of the file
+        :param source_format: source format
+        :param kwargs: additional arguments
+        """
+        if source_format == Format.SQLDUMP_DUCKDB.value or source_format == Format.SQLDUMP_DUCKDB:
+            with self.engine.connect() as conn:
+                sql = text(f"IMPORT DATABASE '{location}'")
+                conn.execute(sql)
+                conn.commit()
+        else:
+            super().import_database(location, source_format=source_format, **kwargs)

{linkml_store-0.1.11 → linkml_store-0.1.13}/src/linkml_store/api/stores/mongodb/mongodb_database.py

@@ -1,7 +1,8 @@
 # mongodb_database.py
 
 import logging
-from typing import Optional
+from pathlib import Path
+from typing import Optional, Union
 
 from pymongo import MongoClient
 from pymongo.database import Database as NativeDatabase
@@ -9,6 +10,9 @@ from pymongo.database import Database as NativeDatabase
 from linkml_store.api import Database
 from linkml_store.api.queries import Query, QueryResult
 from linkml_store.api.stores.mongodb.mongodb_collection import MongoDBCollection
+from linkml_store.utils.file_utils import safe_remove_directory
+from linkml_store.utils.format_utils import Format
+from linkml_store.utils.mongodb_utils import import_mongodb
 
 logger = logging.getLogger(__name__)
 
@@ -27,6 +31,8 @@ class MongoDBDatabase(Database):
     def __init__(self, handle: Optional[str] = None, **kwargs):
         if handle is None:
             handle = "mongodb://localhost:27017/test"
+        if handle == "mongodb":
+            handle = "mongodb://localhost:27017/temporary"
         super().__init__(handle=handle, **kwargs)
 
     @property
@@ -77,3 +83,27 @@ class MongoDBDatabase(Database):
             if collection_name not in self._collections:
                 collection = MongoDBCollection(name=collection_name, parent=self)
                 self._collections[collection_name] = collection
+
+    def export_database(self, location: str, target_format: Optional[Union[str, Format]] = None, **kwargs):
+        if target_format == Format.DUMP_MONGODB.value or target_format == Format.DUMP_MONGODB:
+            path = Path(location)
+            if path.exists():
+                safe_remove_directory(path, no_backup=True)
+            from linkml_store.utils.mongodb_utils import export_mongodb
+
+            export_mongodb(self.handle, location)
+        else:
+            super().export_database(location, target_format=target_format, **kwargs)
+
+    def import_database(self, location: str, source_format: Optional[str] = None, **kwargs):
+        """
+        Import a database from a file or location.
+
+        :param location: location of the file
+        :param source_format: source format
+        :param kwargs: additional arguments
+        """
+        if source_format == Format.DUMP_MONGODB.value or source_format == Format.DUMP_MONGODB:
+            import_mongodb(self.handle, location, drop=True)
+        else:
+            super().import_database(location, source_format=source_format, **kwargs)