linkml-store 0.0.0__py3-none-any.whl

linkml_store/api/stores/mongodb/mongodb_database.py

@@ -0,0 +1,112 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from linkml_runtime import SchemaView
+from linkml_runtime.linkml_model import SlotDefinition
+from linkml_runtime.utils.schema_builder import SchemaBuilder
+from pymongo import MongoClient
+
+from linkml_store.api import Database
+from linkml_store.api.queries import Query, QueryResult
+from linkml_store.api.stores.mongodb.mongodb_collection import MongoDBCollection
+
+
+@dataclass
+class MongoDBDatabase(Database):
+    """
+    A wrapper around a MongoDB database
+    """
+
+    _client: MongoClient = None
+    _database = None
+
+    def __post_init__(self):
+        if not self.handle:
+            self.handle = "mongodb://localhost:27017"
+
+    @property
+    def client(self) -> MongoClient:
+        if not self._client:
+            self._client = MongoClient(self.handle)
+        return self._client
+
+    @property
+    def database(self):
+        if not self._database:
+            db_name = self.handle.split("/")[-1]
+            self._database = self.client[db_name]
+        return self._database
+
+    def commit(self, **kwargs):
+        pass
+
+    def close(self, **kwargs):
+        self.client.close()
+
+    def query(self, query: Query, **kwargs) -> QueryResult:
+        collection = self.database[query.from_table]
+        where_clause = query.where_clause or {}
+        cursor = collection.find(where_clause)
+        if query.limit:
+            cursor = cursor.limit(query.limit)
+        if query.offset:
+            cursor = cursor.skip(query.offset)
+        if query.sort_by:
+            sort_key = [(col, 1) for col in query.sort_by]
+            cursor = cursor.sort(sort_key)
+        rows = list(cursor)
+        num_rows = len(rows)
+        qr = QueryResult(query=query, num_rows=num_rows, rows=rows)
+        return qr
+
+    def init_collections(self):
+        if self._collections is None:
+            self._collections = {}
+        for collection_name in self.database.list_collection_names():
+            if collection_name not in self._collections:
+                collection = MongoDBCollection(name=collection_name, parent=self)
+                self._collections[collection_name] = collection
+
+    def create_collection(self, name: str, alias: Optional[str] = None, **kwargs) -> MongoDBCollection:
+        collection = MongoDBCollection(name=name, parent=self)
+        if not self._collections:
+            self._collections = {}
+        if not alias:
+            alias = name
+        self._collections[alias] = collection
+        return collection
+
+    def induce_schema_view(self) -> SchemaView:
+        sb = SchemaBuilder()
+        schema = sb.schema
+        collection_names = self.database.list_collection_names()
+        for collection_name in collection_names:
+            sb.add_class(collection_name)
+            collection = self.database[collection_name]
+            sample_doc = collection.find_one()
+            if sample_doc:
+                for key, value in sample_doc.items():
+                    if key == "_id":
+                        continue
+                    if isinstance(value, list):
+                        multivalued = True
+                        if value:
+                            value = value[0]
+                        else:
+                            value = None
+                    else:
+                        multivalued = False
+                    if isinstance(value, str):
+                        rng = "string"
+                    elif isinstance(value, int):
+                        rng = "integer"
+                    elif isinstance(value, float):
+                        rng = "float"
+                    elif isinstance(value, bool):
+                        rng = "boolean"
+                    else:
+                        rng = "string"
+                    sd = SlotDefinition(key, range=rng, multivalued=multivalued)
+                    sb.schema.classes[collection_name].attributes[sd.name] = sd
+        sb.add_defaults()
+        return SchemaView(schema)

linkml_store/constants.py

@@ -0,0 +1,7 @@
+import pystow
+
+__all__ = [
+    "LINKML_STORE_MODULE",
+]
+
+LINKML_STORE_MODULE = pystow.module("linkml", "store")

linkml_store/index/implementations/llm_index.py

@@ -0,0 +1,44 @@
+from typing import TYPE_CHECKING, List
+
+import numpy as np
+
+from linkml_store.index.index import INDEX_ITEM, Index
+
+if TYPE_CHECKING:
+    import llm
+
+
+class LLMIndex(Index):
+    """
+    A implementations index wraps the llm library
+    """
+
+    embedding_model_name: str = "ada-002"
+    _embedding_model: "llm.EmbeddingModel" = None
+
+    @property
+    def embedding_model(self):
+        import llm
+
+        if self._embedding_model is None:
+            self._embedding_model = llm.get_embedding_model(self.embedding_model_name)
+        return self._embedding_model
+
+    def text_to_vector(self, text: str) -> INDEX_ITEM:
+        """
+        Convert a text to an indexable object
+
+        :param text:
+        :return:
+        """
+        return self.texts_to_vectors([text])[0]
+
+    def texts_to_vectors(self, texts: List[str]) -> List[INDEX_ITEM]:
+        """
+        Use LLM to embed
+
+        :param texts:
+        :return:
+        """
+        embeddings = self.embedding_model.embed_multi(texts)
+        return [np.array(v, dtype=float) for v in embeddings]

linkml_store/index/implementations/simple_index.py

@@ -0,0 +1,40 @@
+import hashlib
+
+import numpy as np
+
+from linkml_store.index.index import INDEX_ITEM, Index
+
+
+class SimpleIndex(Index):
+    """
+    A implementations index that uses a hash function to generate an index from text.
+
+    This uses a naive method to generate an index from text. It is not suitable for production use.
+    """
+
+    def text_to_vector(self, text: str) -> INDEX_ITEM:
+        """
+        This is a naive method purely for testing
+
+        :param text:
+        :return:
+        """
+        vector_length = self.vector_default_length
+        text = text.lower()
+        # trigrams
+        words = [text[i : i + 3] for i in range(len(text) - 2)]
+
+        vector = np.zeros(vector_length, dtype=float)
+
+        # Iterate over each trigram in the text
+        for word in words:
+            # Generate a hash value for the word
+            hash_value = int(hashlib.sha1(word.encode("utf-8")).hexdigest(), 16)
+
+            # Compute the index in the vector using modulo
+            index = hash_value % vector_length
+
+            # Increment the count at the computed index
+            vector[index] += 1.0
+
+        return vector

linkml_store/index/index.py

@@ -0,0 +1,109 @@
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+import numpy as np
+from pydantic import BaseModel
+
+INDEX_ITEM = np.ndarray
+
+
+def cosine_similarity(vector1, vector2):
+    dot_product = np.dot(vector1, vector2)
+    norm1 = np.linalg.norm(vector1)
+    norm2 = np.linalg.norm(vector2)
+    return dot_product / (norm1 * norm2)
+
+
+class Index(BaseModel):
+    """
+    An index operates on a collection in order to search for objects.
+    """
+
+    name: Optional[str] = None
+    index_function: Optional[Callable] = None
+    distance_function: Optional[Callable] = None
+    index_attributes: Optional[str] = None
+    text_template: Optional[str] = None
+    filter_nulls: Optional[bool] = True
+    vector_default_length: Optional[int] = 1000
+    index_field: Optional[str] = "__index__"
+
+    def object_to_vector(self, obj: Dict[str, Any]) -> INDEX_ITEM:
+        """
+        Convert an object to an indexable object
+
+        :param obj:
+        :return:
+        """
+        return self.text_to_vector(self.object_to_text(obj))
+
+    def objects_to_vectors(self, objs: List[Dict[str, Any]]) -> List[INDEX_ITEM]:
+        """
+        Convert a list of objects to indexable objects
+
+        :param objs:
+        :return:
+        """
+        return [self.object_to_vector(obj) for obj in objs]
+
+    def texts_to_vectors(self, texts: List[str]) -> List[INDEX_ITEM]:
+        """
+        Convert a list of texts to indexable objects
+
+        :param texts:
+        :return:
+        """
+        return [self.text_to_vector(text) for text in texts]
+
+    def text_to_vector(self, text: str) -> INDEX_ITEM:
+        """
+        Convert a text to an indexable object
+
+        :param text:
+        :return:
+        """
+        raise NotImplementedError
+
+    def object_to_text(self, obj: Dict[str, Any]) -> str:
+        """
+        Create a text from an object suitable for indexing.
+        """
+        if self.index_attributes:
+            obj = {k: v for k, v in obj.items() if k in self.index_attributes}
+        if self.filter_nulls:
+            obj = {k: v for k, v in obj.items() if v is not None}
+        if self.text_template:
+            return self.text_template.format(**obj)
+        return str(obj)
+
+    def search(
+        self, query: str, vectors: List[Tuple[str, INDEX_ITEM]], limit: Optional[int] = None
+    ) -> List[Tuple[float, str]]:
+        """
+        Search the index for a query string
+
+        :param query: The query string to search for
+        :param vectors: A list of indexed items, where each item is a tuple of (id, vector)
+        :param limit: The maximum number of results to return (optional)
+        :return: A list of item IDs that match the query
+        """
+
+        # Convert the query string to a vector
+        query_vector = self.text_to_vector(query)
+
+        distances = []
+
+        # Iterate over each indexed item
+        for item_id, item_vector in vectors:
+            # Calculate the Euclidean distance between the query vector and the item vector
+            # distance = 1-np.linalg.norm(query_vector - item_vector)
+            distance = cosine_similarity(query_vector, item_vector)
+            distances.append((distance, item_id))
+
+        # Sort the distances in ascending order
+        distances.sort(key=lambda x: -x[0])
+
+        # Limit the number of results if specified
+        if limit is not None:
+            distances = distances[:limit]
+
+        return distances

linkml_store/utils/io.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+from typing import Iterable, Iterator, Optional, TextIO, Union
+
+from linkml_runtime import SchemaView
+
+from linkml_store.api.collection import OBJECT
+
+
+def export_objects(
+    objects: Iterable[OBJECT],
+    location: Union[Path, str, TextIO],
+    output_type: Optional[str],
+    target_class: Optional[str] = None,
+    schema_view: Optional[SchemaView] = None,
+    **kwargs,
+):
+    """
+    Export objects to a file or stream
+
+    :param objects: objects to export
+    :param location: location to export to
+    :param kwargs:
+    :return:
+    """
+    raise NotImplementedError
+
+
+def import_objects_iter(
+    location: Union[Path, str, TextIO], schema_view: Optional[SchemaView] = None, **kwargs
+) -> Iterator[OBJECT]:
+    """
+    Import objects from a file or stream
+
+    :param location:
+    :param kwargs:
+    :return:
+    """
+    raise NotImplementedError

linkml_store/utils/sql_utils.py

@@ -0,0 +1,126 @@
+import logging
+from typing import Optional, Tuple, Type, Union
+
+import sqlalchemy
+import sqlalchemy.sql.sqltypes as sqlt
+from linkml_runtime.linkml_model import SchemaDefinition, SlotDefinition
+from linkml_runtime.utils.schema_builder import SchemaBuilder
+from sqlalchemy import MetaData
+
+from linkml_store.api.queries import Query
+
+logger = logging.getLogger(__name__)
+
+TYPE_MAP = {
+    sqlt.TEXT: "string",
+    sqlt.INTEGER: "integer",
+    sqlt.FLOAT: "float",
+}
+
+
+def _map_type(typ: Type) -> str:
+    for k, v in TYPE_MAP.items():
+        if isinstance(typ, k):
+            return v
+    return "string"
+
+
+def where_clause_to_sql(query: Query) -> str:
+    if not query.where_clause:
+        return ""
+    if isinstance(query.where_clause, str):
+        where_clause_sql = query.where_clause
+    elif isinstance(query.where_clause, list):
+        where_clause_sql = " AND ".join(query.where_clause)
+    elif isinstance(query.where_clause, dict):
+        # TODO: bobby tables
+        where_clause_sql = " AND ".join([f"{k} = '{v}'" for k, v in query.where_clause.items()])
+    else:
+        raise ValueError(f"Invalid where_clause type: {type(query.where_clause)}")
+    return "WHERE " + where_clause_sql
+
+
+def query_to_sql(query: Query, count=False, limit=None, offset: Optional[int] = None):
+    select_cols = query.select_cols if query.select_cols else ["*"]
+    if count:
+        sql_str = ["SELECT COUNT(*)"]
+    else:
+        sql_str = [f"SELECT {', '.join(select_cols)}"]
+    sql_str.append(f"FROM {query.from_table}")
+    sql_str.append(where_clause_to_sql(query))
+    if not count:
+        if query.sort_by:
+            sql_str.append(f"ORDER BY {', '.join(query.sort_by)}")
+    if not count:
+        if limit is None:
+            limit = query.limit
+        if limit is None:
+            limit = 100
+        if limit < 0:
+            limit = None
+        if limit is not None:
+            sql_str.append(f" LIMIT {limit}")
+        offset = offset if offset else query.offset
+        if offset:
+            sql_str.append(f" OFFSET {offset}")
+    sql_str = [line for line in sql_str if line]
+    return "\n".join(sql_str)
+
+
+def facet_count_sql(query: Query, facet_column: Union[str, Tuple[str, ...]], multivalued=False) -> str:
+    # Create a modified WHERE clause that excludes conditions directly related to facet_column
+    modified_where = None
+    if query.where_clause:
+        where_clause_sql = where_clause_to_sql(query)
+        # Split the where clause into conditions and exclude those related to the facet_column
+        conditions = [cond for cond in where_clause_sql.split(" AND ") if not cond.startswith(f"{facet_column} ")]
+        modified_where = " AND ".join(conditions)
+
+    if isinstance(facet_column, tuple):
+        if multivalued:
+            raise NotImplementedError("Multivalued facets are not supported for multiple columns")
+        facet_column = ", ".join(facet_column)
+    from_table = query.from_table
+    if multivalued:
+        from_table = f"(SELECT UNNEST({facet_column}) as {facet_column} FROM {query.from_table})"
+    sql_str = [f"SELECT {facet_column}, COUNT(*) as count", f"FROM {from_table}"]
+    if modified_where:
+        sql_str.append(f"{modified_where}")
+    sql_str.append(f"GROUP BY {facet_column}")
+    sql_str.append("ORDER BY count DESC")  # Optional, order by count for convenience
+    return "\n".join(sql_str)
+
+
+def introspect_schema(engine: sqlalchemy.Engine) -> SchemaDefinition:
+    """
+    Introspect a database schema and return a SchemaDefinition object
+
+    :param engine:
+    :return:
+    """
+    metadata_obj = MetaData()
+    logging.info(f"Reflecting using {engine}")
+    metadata_obj.reflect(bind=engine)
+    sb = SchemaBuilder()
+    schema = sb.schema
+    for table in metadata_obj.sorted_tables:
+        logging.info(f"Importing {table.name}")
+        sb.add_class(table.name)
+        cls = schema.classes[table.name]
+        pks = [column for column in table.columns if column.primary_key]
+        if len(pks) == 1:
+            pk = pks.pop().name
+        else:
+            pk = None
+        for column in table.columns:
+            slot = SlotDefinition(column.name)
+            cls.attributes[slot.name] = slot
+            if pk and pk == column.name:
+                slot.identifier = True
+            if column.foreign_keys:
+                for fk in column.foreign_keys:
+                    [fk_table, fk_table_col] = str(fk.column).split(".")
+                    slot.range = fk_table
+            else:
+                slot.range = _map_type(column.type)
+    return schema

linkml_store-0.0.0.dist-info/LICENSE

@@ -0,0 +1,22 @@
+
+The MIT License (MIT)
+
+Copyright (c) 2024 Monarch Initiative
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

linkml_store-0.0.0.dist-info/METADATA

@@ -0,0 +1,44 @@
+Metadata-Version: 2.1
+Name: linkml-store
+Version: 0.0.0
+Summary: linkml-store
+License: MIT
+Author: Author 1
+Author-email: author@org.org
+Requires-Python: >=3.9, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*, !=3.8.*
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: analytics
+Provides-Extra: app
+Provides-Extra: llm
+Provides-Extra: mongodb
+Provides-Extra: tests
+Requires-Dist: black (>=24.0.0) ; extra == "tests"
+Requires-Dist: click
+Requires-Dist: duckdb (>=0.10.1,<0.11.0)
+Requires-Dist: duckdb-engine (>=0.11.2,<0.12.0)
+Requires-Dist: linkml-runtime (>=1.7.5,<2.0.0)
+Requires-Dist: llm ; extra == "llm"
+Requires-Dist: matplotlib ; extra == "analytics"
+Requires-Dist: pandas (>=2.2.1,<3.0.0) ; extra == "analytics"
+Requires-Dist: plotly ; extra == "analytics"
+Requires-Dist: pydantic (>=2.0.0,<3.0.0)
+Requires-Dist: pymongo ; extra == "mongodb"
+Requires-Dist: pystow (>=0.5.4,<0.6.0)
+Requires-Dist: seaborn ; extra == "analytics"
+Requires-Dist: sqlalchemy
+Requires-Dist: streamlit (>=1.32.2,<2.0.0) ; extra == "app"
+Description-Content-Type: text/markdown
+
+# linkml-store
+
+This is the project description.
+
+# Acknowledgements
+
+This [cookiecutter](https://cookiecutter.readthedocs.io/en/stable/README.html) project was developed from the [monarch-project-template](https://github.com/monarch-initiative/monarch-project-template) template and will be kept up-to-date using [cruft](https://cruft.github.io/cruft/).
+

linkml_store-0.0.0.dist-info/RECORD

@@ -0,0 +1,29 @@
+linkml_store/__init__.py,sha256=jlU6WOUAn8cKIhzbTULmBTWpW9gZdEt7q_RI6KZN1bY,118
+linkml_store/api/__init__.py,sha256=Il3wGkN6evgc3OOWQYL7q3J-tRBs_0yQ3Nj-s6eo-0Y,284
+linkml_store/api/client.py,sha256=6zGzhTd6zJ-V5fzLbgljI6onq_XVvaxoeyXK5d1U9wM,4353
+linkml_store/api/collection.py,sha256=zr9iMXCJPsBOYLwO8I0dLIl01sVG4sgXtpgVdHJIVNo,11334
+linkml_store/api/database.py,sha256=oulUxyUGfq6dMQge___dYcevunDOIlLUBGTeewRZqDs,6682
+linkml_store/api/metadata.py,sha256=k9F6D_nuIZ0wWocj0ew2FYSKOc06CJWvwoUpHcrs7JA,69
+linkml_store/api/queries.py,sha256=5WgI_od_Qlpiza-u-XCrxL0F3Etf6mdeCCSSxeHT0PI,1698
+linkml_store/api/stores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+linkml_store/api/stores/duckdb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+linkml_store/api/stores/duckdb/duckdb_collection.py,sha256=3hxwnRoY8pIZuW09yi4BSRDUT_pzTo9Mj82SBTT1aNw,4194
+linkml_store/api/stores/duckdb/duckdb_database.py,sha256=efTwzAoAc4YugBqNl9aQhlBzcPU4GDLwGQC5UPk_Uuw,6104
+linkml_store/api/stores/duckdb/mappings.py,sha256=S4MWetLpQcxOwwedXrZTqazxdaHIQXXbq4VRq9Ok4B4,123
+linkml_store/api/stores/mongodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+linkml_store/api/stores/mongodb/mongodb_collection.py,sha256=atFpJWTENwCBmRjtC5iWkwwV-3XnVr16A_nuXDS76Ig,2094
+linkml_store/api/stores/mongodb/mongodb_database.py,sha256=EpiTUftXlmOC8V4lPcrGOX2xq0leqWpmPVaZGsUMESI,3984
+linkml_store/constants.py,sha256=x4ZmDsfE9rZcL5WpA93uTKrRWzCD6GodYXviVzIvR38,112
+linkml_store/index/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+linkml_store/index/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+linkml_store/index/implementations/llm_index.py,sha256=JAxg1dIuU7W8ZtnsGcogbBUvyrPTZCbKhEFv-0lEKrk,1061
+linkml_store/index/implementations/simple_index.py,sha256=ep73dg86QqV7B7t_VoOK1weVx5RD0xvJ7uJPsszd5I0,1134
+linkml_store/index/index.py,sha256=uwgKtw76ch0qe5CbSj2Ft_WAhY94Qm5qLBJucl-VPWA,3420
+linkml_store/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+linkml_store/utils/io.py,sha256=JHUrWDtlZC2jtN_PQZ4ypdGIyYlftZEN3JaCvEPs44w,884
+linkml_store/utils/sql_utils.py,sha256=sCM9GqEW6zZ3-2n5tsW7gzi0DerP6TXx3AJDY_0kCJ4,4557
+linkml_store-0.0.0.dist-info/LICENSE,sha256=77mDOslUnalYnuq9xQYZKtIoNEzcH9mIjvWHOKjamnE,1086
+linkml_store-0.0.0.dist-info/METADATA,sha256=Ignb5ab4IwjIfm60g2yciUqYBb0IstFvABlOMFTygPk,1717
+linkml_store-0.0.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+linkml_store-0.0.0.dist-info/entry_points.txt,sha256=YdXo7B96u7fP8WX9cirGelqH0kjx7vGjT-w8hq8HFSE,54
+linkml_store-0.0.0.dist-info/RECORD,,

linkml_store-0.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.0
+Root-Is-Purelib: true
+Tag: py3-none-any

linkml_store-0.0.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+linkml-store=linkml_store.cli:main
+