linkml-store 0.1.10__tar.gz → 0.1.11__tar.gz

{linkml_store-0.1.10 → linkml_store-0.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: linkml-store
-Version: 0.1.10
+Version: 0.1.11
 Summary: linkml-store
 License: MIT
 Author: Author 1
@@ -16,6 +16,7 @@ Provides-Extra: analytics
 Provides-Extra: app
 Provides-Extra: chromadb
 Provides-Extra: fastapi
+Provides-Extra: frictionless
 Provides-Extra: h5py
 Provides-Extra: llm
 Provides-Extra: map
@@ -30,11 +31,12 @@ Requires-Dist: click
 Requires-Dist: duckdb (>=0.10.1,<0.11.0)
 Requires-Dist: duckdb-engine (>=0.11.2)
 Requires-Dist: fastapi ; extra == "fastapi"
+Requires-Dist: frictionless ; extra == "frictionless"
 Requires-Dist: h5py ; extra == "h5py"
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: jsonlines (>=4.0.0,<5.0.0)
-Requires-Dist: linkml ; extra == "validation"
-Requires-Dist: linkml-runtime (>=1.8.0rc2)
+Requires-Dist: linkml (>=1.8.0) ; extra == "validation"
+Requires-Dist: linkml-runtime (>=1.8.0)
 Requires-Dist: linkml_map ; extra == "map"
 Requires-Dist: linkml_renderer ; extra == "renderer"
 Requires-Dist: llm ; extra == "llm"
@@ -122,6 +124,10 @@ need to have a vector database to run embedding search!
 
 See [How to Use-Semantic-Search](https://linkml.io/linkml-store/how-to/Use-Semantic-Search.html)
 
+### Use with LLMs
+
+TODO - docs
+
 ### Validation
 
 LinkML-Store is backed by [LinkML](https://linkml.io), which allows
@@ -131,6 +137,33 @@ See [Indexing JSON](https://linkml.io/linkml-store/how-to/Index-Phenopackets.htm
 
 and [Referential Integrity](https://linkml.io/linkml-store/how-to/Check-Referential-Integrity.html)
 
+## Web API
+
+There is a preliminary API following HATEOAS principles implemented using FastAPI.
+
+To start you should first create a config file, e.g. `db/conf.yaml`:
+
+Then run:
+
+```
+export LINKML_STORE_CONFIG=./db/conf.yaml
+make api
+```
+
+The API returns links as well as data objects, it's recommended to use a Chrome plugin for JSON viewing
+for exploring the API. TODO: add docs here.
+
+The main endpoints are:
+
+* `http://localhost:8000/` - the root of the API
+* `http://localhost:8000/pages/` - browse the API via HTML
+* `http://localhost:8000/docs` - the Swagger UI
+
+## Streamlit app
+
+```
+make app
+```
 
 ## Background
 

{linkml_store-0.1.10 → linkml_store-0.1.11}/README.md

@@ -69,6 +69,10 @@ need to have a vector database to run embedding search!
 
 See [How to Use-Semantic-Search](https://linkml.io/linkml-store/how-to/Use-Semantic-Search.html)
 
+### Use with LLMs
+
+TODO - docs
+
 ### Validation
 
 LinkML-Store is backed by [LinkML](https://linkml.io), which allows
@@ -78,6 +82,33 @@ See [Indexing JSON](https://linkml.io/linkml-store/how-to/Index-Phenopackets.htm
 
 and [Referential Integrity](https://linkml.io/linkml-store/how-to/Check-Referential-Integrity.html)
 
+## Web API
+
+There is a preliminary API following HATEOAS principles implemented using FastAPI.
+
+To start you should first create a config file, e.g. `db/conf.yaml`:
+
+Then run:
+
+```
+export LINKML_STORE_CONFIG=./db/conf.yaml
+make api
+```
+
+The API returns links as well as data objects, it's recommended to use a Chrome plugin for JSON viewing
+for exploring the API. TODO: add docs here.
+
+The main endpoints are:
+
+* `http://localhost:8000/` - the root of the API
+* `http://localhost:8000/pages/` - browse the API via HTML
+* `http://localhost:8000/docs` - the Swagger UI
+
+## Streamlit app
+
+```
+make app
+```
 
 ## Background
 

{linkml_store-0.1.10 → linkml_store-0.1.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "linkml-store"
-version = "0.1.10"
+version = "0.1.11"
 description = "linkml-store"
 authors = ["Author 1 <author@org.org>"]
 license = "MIT"
@@ -10,7 +10,7 @@ readme = "README.md"
 python = "^3.9, !=3.9.7"
 click = "*"
 pydantic = "^2.0.0"
-linkml-runtime = ">=1.8.0rc2"
+linkml-runtime = ">=1.8.0"
 streamlit = { version = "^1.32.2", optional = true }
 sqlalchemy = "*"
 duckdb = "^0.10.1"
@@ -25,9 +25,10 @@ pymongo = { version="*", optional = true }
 chromadb = { version="*", optional = true }
 pyarrow = { version="*", optional = true }
 h5py = { version="*", optional = true }
-linkml = { version="*", optional = true }
+linkml = { version=">=1.8.0", optional = true }
 linkml_map = { version="*", optional = true }
 linkml_renderer = { version="*", optional = true }
+frictionless = { version="*", optional = true }
 pandas = ">=2.2.1"
 jinja2 = "^3.1.4"
 jsonlines = "^4.0.0"
@@ -49,6 +50,7 @@ nbsphinx = "*"
 jupyter = "*"
 jupysql = "*"
 papermill = "*"
+nbdime = "*"
 
 [tool.poetry.group.tests.dependencies]
 pytest = "^7.4.0"
@@ -71,6 +73,7 @@ validation = ["linkml"]
 map = ["linkml_map"]
 renderer = ["linkml_renderer"]
 fastapi = ["fastapi", "uvicorn"]
+frictionless = ["frictionless"]
 
 [tool.poetry.scripts]
 linkml-store = "linkml_store.cli:cli"

{linkml_store-0.1.10 → linkml_store-0.1.11}/src/linkml_store/api/client.py

@@ -242,7 +242,7 @@ class Client:
         Return all attached databases
 
         Examples
-        --------
+
         >>> client = Client()
         >>> _ = client.attach_database("duckdb", alias="test1")
         >>> _ = client.attach_database("duckdb", alias="test2")
@@ -268,25 +268,81 @@ class Client:
         """
         Drop a database.
 
+        Example (in-memory):
+
+        >>> client = Client()
+        >>> db1 = client.attach_database("duckdb", alias="test1")
+        >>> db2 = client.attach_database("duckdb", alias="test2")
+        >>> len(client.databases)
+        2
+        >>> client.drop_database("test1")
+        >>> len(client.databases)
+        1
+
+        Databases that persist on disk:
+
+        >>> client = Client()
+        >>> path = Path("tmp/test.db")
+        >>> path.parent.mkdir(parents=True, exist_ok=True)
+        >>> db = client.attach_database(f"duckdb:///{path}", alias="test")
+        >>> len(client.databases)
+        1
+        >>> db.store({"persons": [{"id": "P1", "name": "John"}]})
+        >>> db.commit()
+        >>> Path("tmp/test.db").exists()
+        True
+        >>> client.drop_database("test")
+        >>> len(client.databases)
+        0
+        >>> Path("tmp/test.db").exists()
+        False
+
+        Dropping a non-existent database:
+
+        >>> client = Client()
+        >>> client.drop_database("duckdb:///tmp/made-up1", missing_ok=True)
+        >>> client.drop_database("duckdb:///tmp/made-up2", missing_ok=False)
+        Traceback (most recent call last):
+        ...
+        ValueError: Database duckdb:///tmp/made-up2 not found
+
         :param name:
         :param missing_ok:
         :return:
         """
-        if name in self._databases:
-            db = self._databases[name]
-            db.drop(**kwargs)
-            del self._databases[name]
+        if self._databases:
+            if name in self._databases:
+                db = self._databases[name]
+                db.drop(**kwargs)
+                del self._databases[name]
+            else:
+                if not missing_ok:
+                    raise ValueError(f"Database {name} not found")
         else:
-            if not missing_ok:
-                raise ValueError(f"Database {name} not found")
+            db = self.get_database(name, create_if_not_exists=True)
+            db.drop(**kwargs)
 
     def drop_all_databases(self, **kwargs):
         """
         Drop all databases.
 
+        Example (in-memory):
+
+        >>> client = Client()
+        >>> db1 = client.attach_database("duckdb", alias="test1")
+        >>> assert "test1" in client.databases
+        >>> db2 = client.attach_database("duckdb", alias="test2")
+        >>> assert "test2" in client.databases
+        >>> client.drop_all_databases()
+        >>> len(client.databases)
+        0
+
+
         :param missing_ok:
         :return:
         """
+        if not self._databases:
+            return
         for name in list(self._databases.keys()):
             self.drop_database(name, missing_ok=False, **kwargs)
         self._databases = {}

{linkml_store-0.1.10 → linkml_store-0.1.11}/src/linkml_store/api/collection.py

@@ -14,7 +14,7 @@ from pydantic import BaseModel
 
 from linkml_store.api.types import DatabaseType
 from linkml_store.index import get_indexer
-from linkml_store.utils.format_utils import load_objects
+from linkml_store.utils.format_utils import load_objects, load_objects_from_url
 from linkml_store.utils.object_utils import clean_empties
 from linkml_store.utils.patch_utils import PatchDict, apply_patches_to_list, patches_from_objects_lists
 
@@ -61,6 +61,7 @@ class Collection(Generic[DatabaseType]):
     # name: str
     parent: Optional[DatabaseType] = None
     _indexers: Optional[Dict[str, Indexer]] = None
+    _initialized: Optional[bool] = None
     # hidden: Optional[bool] = False
 
     metadata: Optional[CollectionConfig] = None
@@ -73,7 +74,7 @@ class Collection(Generic[DatabaseType]):
         if metadata:
             self.metadata = metadata
         else:
-            self.metadata = CollectionConfig(name=name, **kwargs)
+            self.metadata = CollectionConfig(type=name, **kwargs)
             if not self.metadata.alias:
                 self.metadata.alias = name
             if not self.metadata.type:
@@ -81,17 +82,6 @@ class Collection(Generic[DatabaseType]):
         # if name is not None and self.metadata.name is not None and name != self.metadata.name:
         #    raise ValueError(f"Name mismatch: {name} != {self.metadata.name}")
 
-    @property
-    def name(self) -> str:
-        """
-        Return the name of the collection.
-
-        TODO: deprecate in favor of Type
-
-        :return: name of the collection
-        """
-        return self.metadata.name
-
     @property
     def hidden(self) -> bool:
         """
@@ -118,12 +108,18 @@ class Collection(Generic[DatabaseType]):
         >>> collection.target_class_name
         'Person'
 
+        >>> collection = db.create_collection("Organization")
+        >>> collection.target_class_name
+        'Organization'
+        >>> collection.alias
+        'Organization'
+
         :return: name of the class which members of this collection instantiate
         """
         # TODO: this is a shim layer until we can normalize on this
         if self.metadata.type:
             return self.metadata.type
-        return self.name
+        return self.alias
 
     @property
     def alias(self):
@@ -161,10 +157,9 @@ class Collection(Generic[DatabaseType]):
         :return:
         """
         # TODO: this is a shim layer until we can normalize on this
-        # TODO: this is a shim layer until we can normalize on this
         if self.metadata.alias:
             return self.metadata.alias
-        return self.name
+        return self.target_class_name
 
     def replace(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
         """
@@ -201,7 +196,14 @@ class Collection(Generic[DatabaseType]):
         """
         raise NotImplementedError
 
+    def _pre_query_hook(self, query: Optional[Query] = None, **kwargs):
+        logger.info(f"Pre-query hook (state: {self._initialized}; Q= {query}")
+        if not self._initialized:
+            self._materialize_derivations()
+            self._initialized = True
+
     def _post_insert_hook(self, objs: List[OBJECT], **kwargs):
+        self._initialized = True
         patches = [{"op": "add", "path": "/0", "value": obj} for obj in objs]
         self._broadcast(patches, **kwargs)
 
@@ -305,6 +307,7 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
+        self._pre_query_hook()
         return self.parent.query(query, **kwargs)
 
     def query_facets(
@@ -340,7 +343,6 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
-        # TODO
         id_field = self.identifier_attribute_name
         if not id_field:
             raise ValueError(f"No identifier for {self.name}")
@@ -399,9 +401,10 @@ class Collection(Generic[DatabaseType]):
         :return:
         """
         query = self._create_query(where_clause=where)
+        self._pre_query_hook(query)
         return self.query(query, **kwargs)
 
-    def find_iter(self, where: Optional[Any] = None, **kwargs) -> Iterator[OBJECT]:
+    def find_iter(self, where: Optional[Any] = None, page_size=100, **kwargs) -> Iterator[OBJECT]:
         """
         Find objects in the collection using a where query.
 
@@ -409,9 +412,22 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
-        qr = self.find(where=where, limit=-1, **kwargs)
-        for row in qr.rows:
-            yield row
+        total_rows = None
+        offset = 0
+        if page_size < 1:
+            raise ValueError(f"Invalid page size: {page_size}")
+        while True:
+            qr = self.find(where=where, offset=offset, limit=page_size, **kwargs)
+            if total_rows is None:
+                total_rows = qr.num_rows
+            if not qr.rows:
+                return
+            for row in qr.rows:
+                yield row
+            offset += page_size
+            if offset >= total_rows:
+                break
+        return
 
     def search(
         self,
@@ -454,6 +470,7 @@ class Collection(Generic[DatabaseType]):
         :param kwargs:
         :return:
         """
+        self._pre_query_hook()
         if index_name is None:
             if len(self.indexers) == 1:
                 index_name = list(self.indexers.keys())[0]
@@ -494,10 +511,93 @@ class Collection(Generic[DatabaseType]):
             raise ValueError(f"Collection has no alias: {self} // {self.metadata}")
         return self.alias.startswith("internal__")
 
-    def load_from_source(self):
-        objects = load_objects(self.metadata.source_location)
+    def exists(self) -> Optional[bool]:
+        """
+        Check if the collection exists.
+
+        :return:
+        """
+        cd = self.class_definition()
+        return cd is not None
+
+    def load_from_source(self, load_if_exists=False):
+        """
+        Load objects from the source location.
+
+        :param load_if_exists:
+        :return:
+        """
+        if not load_if_exists and self.exists():
+            return
+        metadata = self.metadata
+        if metadata.source:
+            source = metadata.source
+            kwargs = source.arguments or {}
+            if source.local_path:
+                objects = load_objects(
+                    metadata.source.local_path, format=source.format, expected_type=source.expected_type, **kwargs
+                )
+            elif metadata.source.url:
+                objects = load_objects_from_url(
+                    metadata.source.url, format=source.format, expected_type=source.expected_type, **kwargs
+                )
         self.insert(objects)
 
+    def _check_if_initialized(self) -> bool:
+        return self._initialized
+
+    def _materialize_derivations(self, **kwargs):
+        metadata = self.metadata
+        if not metadata.derived_from:
+            logger.info(f"No metadata for {self.alias}; no derivations")
+            return
+        if self._check_if_initialized():
+            logger.info(f"Already initialized {self.alias}; no derivations")
+            return
+        parent_db = self.parent
+        client = parent_db.parent
+        # cd = self.class_definition()
+        for derivation in metadata.derived_from:
+            # TODO: optimize this; utilize underlying engine
+            logger.info(f"Deriving from {derivation}")
+            if derivation.database:
+                db = client.get_database(derivation.database)
+            else:
+                db = parent_db
+            if derivation.collection:
+                coll = db.get_collection(derivation.collection)
+            else:
+                coll = self
+            coll.class_definition()
+            source_obj_iter = coll.find_iter(derivation.where or {})
+            mappings = derivation.mappings
+            if not mappings:
+                raise ValueError(f"No mappings for {self.name}")
+            target_class_name = self.target_class_name
+            from linkml_map.session import Session
+
+            session = Session()
+            session.set_source_schema(db.schema_view.schema)
+            session.set_object_transformer(
+                {
+                    "class_derivations": {
+                        target_class_name: {
+                            "populated_from": coll.target_class_name,
+                            "slot_derivations": mappings,
+                        },
+                    }
+                },
+            )
+            logger.debug(f"Session Spec: {session.object_transformer}")
+            tr_objs = []
+            for source_obj in source_obj_iter:
+                tr_obj = session.transform(source_obj, source_type=coll.target_class_name)
+                tr_objs.append(tr_obj)
+            if not tr_objs:
+                raise ValueError(f"No objects derived from {coll.name}")
+            self.insert(tr_objs)
+            self.commit()
+
     def attach_indexer(self, index: Union[Indexer, str], name: Optional[str] = None, auto_index=True, **kwargs):
         """
         Attach an index to the collection.
@@ -572,7 +672,7 @@ class Collection(Generic[DatabaseType]):
         :param indexer:
         :return:
         """
-        return f"internal__index__{self.name}__{index_name}"
+        return f"internal__index__{self.alias}__{index_name}"
 
     def index_objects(self, objs: List[OBJECT], index_name: str, replace=False, **kwargs):
         """
@@ -638,6 +738,9 @@ class Collection(Generic[DatabaseType]):
         """
         Return the class definition for the collection.
 
+        If no schema has been explicitly set, and the native database does not
+        have a schema, then a schema will be induced from the objects in the collection.
+
         :return:
         """
         sv: SchemaView = self.parent.schema_view
@@ -722,7 +825,9 @@ class Collection(Generic[DatabaseType]):
         else:
             return None
 
-    def induce_class_definition_from_objects(self, objs: List[OBJECT], max_sample_size=10) -> ClassDefinition:
+    def induce_class_definition_from_objects(
+        self, objs: List[OBJECT], max_sample_size: Optional[int] = None
+    ) -> ClassDefinition:
         """
         Induce a class definition from a list of objects.
 
@@ -733,6 +838,9 @@ class Collection(Generic[DatabaseType]):
         :param max_sample_size:
         :return:
         """
+        # TODO: use schemaview
+        if max_sample_size is None:
+            max_sample_size = 10
         if not self.target_class_name:
             raise ValueError(f"No target_class_name for {self.alias}")
         cd = ClassDefinition(self.target_class_name)
@@ -795,6 +903,7 @@ class Collection(Generic[DatabaseType]):
             for other_rng in rngs:
                 if rng != other_rng:
                     raise ValueError(f"Conflict: {rng} != {other_rng} for {vs}")
+            logger.debug(f"Inducing {k} as {rng} {multivalued} {inlined}")
             cd.attributes[k] = SlotDefinition(k, range=rng, multivalued=multivalued, inlined=inlined)
             if exact_dimensions_list:
                 array_expr = ArrayExpression(exact_number_dimensions=len(exact_dimensions_list[0]))
@@ -828,7 +937,7 @@ class Collection(Generic[DatabaseType]):
         """
         Apply a patch to the collection.
 
-        Patches conform to the JSON Patch format,
+        Patches conform to the JSON Patch format.
 
         :param patches:
         :param kwargs:
@@ -841,11 +950,11 @@ class Collection(Generic[DatabaseType]):
         new_objs = apply_patches_to_list(all_objs, patches, primary_key=primary_key, **kwargs)
         self.replace(new_objs)
 
-    def diff(self, other: "Collection", **kwargs):
+    def diff(self, other: "Collection", **kwargs) -> List[PatchDict]:
         """
         Diff two collections.
 
-        :param other:
+        :param other: The collection to diff against
         :param kwargs:
         :return:
         """
@@ -872,8 +981,7 @@ class Collection(Generic[DatabaseType]):
         if not cd:
             raise ValueError(f"Cannot find class definition for {self.target_class_name}")
         class_name = cd.name
-        result = self.find(**kwargs)
-        for obj in result.rows:
+        for obj in self.find_iter(**kwargs):
             obj = clean_empties(obj)
             yield from validator.iter_results(obj, class_name)
 

{linkml_store-0.1.10 → linkml_store-0.1.11}/src/linkml_store/api/config.py

@@ -4,14 +4,43 @@ from pydantic import BaseModel, Field
 
 
 class ConfiguredBaseModel(BaseModel, extra="forbid"):
+    """
+    Base class for all configuration models.
+    """
+
     pass
 
 
+class DerivationConfiguration(ConfiguredBaseModel):
+    """
+    Configuration for a derivation
+    """
+
+    database: Optional[str] = None
+    collection: Optional[str] = None
+    mappings: Optional[Dict[str, Any]] = None
+    where: Optional[Dict[str, Any]] = None
+
+
+class CollectionSource(ConfiguredBaseModel):
+    """
+    Metadata about a source
+    """
+
+    url: Optional[str] = None
+    local_path: Optional[str] = None
+    source_location: Optional[str] = None
+    refresh_interval_days: Optional[float] = None
+    expected_type: Optional[str] = None
+    format: Optional[str] = None
+    arguments: Optional[Dict[str, Any]] = None
+
+
 class CollectionConfig(ConfiguredBaseModel):
-    name: Optional[str] = Field(
-        default=None,
-        description="An optional name for the collection",
-    )
+    """
+    Configuration for a collection
+    """
+
     alias: Optional[str] = Field(
         default=None,
         description="An optional alias for the collection",
@@ -40,13 +69,22 @@ class CollectionConfig(ConfiguredBaseModel):
         default=False,
         description="Whether the collection is prepopulated",
     )
-    source_location: Optional[str] = Field(
+    source: Optional[CollectionSource] = Field(
         default=None,
-        description="Filesystem or remote URL that stores the data",
+        description="Metadata about the source",
+    )
+    # TODO: derived_from
+    derived_from: Optional[List[DerivationConfiguration]] = Field(
+        default=None,
+        description="LinkML-Map derivations",
     )
 
 
 class DatabaseConfig(ConfiguredBaseModel):
+    """
+    Configuration for a database
+    """
+
     handle: str = Field(
         default="duckdb:///:memory:",
         description="The database handle, e.g., 'duckdb:///:memory:' or 'mongodb://localhost:27017'",
@@ -91,6 +129,10 @@ class DatabaseConfig(ConfiguredBaseModel):
 
 
 class ClientConfig(ConfiguredBaseModel):
+    """
+    Configuration for a client
+    """
+
     handle: Optional[str] = Field(
         default=None,
         description="The client handle",