linkml-store 0.0.0__tar.gz

linkml_store-0.0.0/LICENSE

@@ -0,0 +1,22 @@
+
+The MIT License (MIT)
+
+Copyright (c) 2024 Monarch Initiative
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

linkml_store-0.0.0/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.1
+Name: linkml-store
+Version: 0.0.0
+Summary: linkml-store
+License: MIT
+Author: Author 1
+Author-email: author@org.org
+Requires-Python: >=3.9, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*, !=3.8.*
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: analytics
+Provides-Extra: app
+Provides-Extra: llm
+Provides-Extra: mongodb
+Provides-Extra: tests
+Requires-Dist: black (>=24.0.0) ; extra == "tests"
+Requires-Dist: click
+Requires-Dist: duckdb (>=0.10.1,<0.11.0)
+Requires-Dist: duckdb-engine (>=0.11.2,<0.12.0)
+Requires-Dist: linkml-runtime (>=1.7.5,<2.0.0)
+Requires-Dist: llm ; extra == "llm"
+Requires-Dist: matplotlib ; extra == "analytics"
+Requires-Dist: pandas (>=2.2.1,<3.0.0) ; extra == "analytics"
+Requires-Dist: plotly ; extra == "analytics"
+Requires-Dist: pydantic (>=2.0.0,<3.0.0)
+Requires-Dist: pymongo ; extra == "mongodb"
+Requires-Dist: pystow (>=0.5.4,<0.6.0)
+Requires-Dist: seaborn ; extra == "analytics"
+Requires-Dist: sqlalchemy
+Requires-Dist: streamlit (>=1.32.2,<2.0.0) ; extra == "app"
+Description-Content-Type: text/markdown
+
+# linkml-store
+
+This is the project description.
+
+# Acknowledgements
+
+This [cookiecutter](https://cookiecutter.readthedocs.io/en/stable/README.html) project was developed from the [monarch-project-template](https://github.com/monarch-initiative/monarch-project-template) template and will be kept up-to-date using [cruft](https://cruft.github.io/cruft/).
+

linkml_store-0.0.0/README.md

@@ -0,0 +1,7 @@
+# linkml-store
+
+This is the project description.
+
+# Acknowledgements
+
+This [cookiecutter](https://cookiecutter.readthedocs.io/en/stable/README.html) project was developed from the [monarch-project-template](https://github.com/monarch-initiative/monarch-project-template) template and will be kept up-to-date using [cruft](https://cruft.github.io/cruft/).

linkml_store-0.0.0/pyproject.toml

@@ -0,0 +1,124 @@
+[tool.poetry]
+name = "linkml-store"
+version = "0.0.0"
+description = "linkml-store"
+authors = ["Author 1 <author@org.org>"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.9, !=3.9.7"
+click = "*"
+pydantic = "^2.0.0"
+linkml-runtime = "^1.7.5"
+streamlit = { version = "^1.32.2", optional = true }
+sqlalchemy = "*"
+duckdb = "^0.10.1"
+duckdb-engine = "^0.11.2"
+matplotlib = { version = "*", optional = true }
+seaborn = { version = "*", optional = true }
+plotly = { version = "*", optional = true }
+pystow = "^0.5.4"
+black = { version=">=24.0.0", optional = true }
+llm = { version="*", optional = true }
+pymongo = { version="*", optional = true }
+pandas = "^2.2.1"
+
+[tool.poetry.group.dev.dependencies]
+pytest = {version = ">=7.1.2"}
+tox = {version = ">=3.25.1"}
+pre-commit = {version = ">=3.3.3"}
+sphinx = {version = ">=6.1.3"}
+sphinx-rtd-theme = {version = ">=1.0.0"}
+sphinx-autodoc-typehints = {version = "<2.0.0"}
+sphinx-click = {version = ">=4.3.0"}
+myst-parser = {version = ">=0.18.1"}
+nbsphinx = "*"
+jupyter = "*"
+
+[tool.poetry.group.tests.dependencies]
+pytest = "^7.4.0"
+pytest-subtests = "^0.11.0"
+numpy = [
+  { "version" = ">=1.24.3", "python" = "<3.12" },
+  { "version" = ">=1.25.2", "python" = ">=3.12" }
+]
+
+[tool.poetry.extras]
+analytics = ["pandas", "matplotlib", "seaborn", "plotly"]
+app = ["streamlit"]
+tests = ["black"]
+llm = ["llm"]
+mongodb = ["pymongo"]
+
+[tool.poetry.scripts]
+linkml-store = "linkml_store.cli:main"
+
+[tool.poetry-dynamic-versioning]
+enable = false
+vcs = "git"
+style = "pep440"
+
+[tool.black]
+line-length = 120
+target-version = ["py38", "py39", "py310", "py311"]
+force-exclude = '''
+/(
+  # default exclude
+  \.direnv|\.eggs|\.git|\.hg|\.ipynb_checkpoints|\.mypy_cache|\.nox|\.pytest_cache|\.ruff_cache|\.tox|\.svn|\.venv|\.vscode|__pypackages__|_build|buck-out|build|dist|venv
+  # additional exclude
+  | tests.*/output
+  | __snapshots__
+  | docs
+  | examples
+  | notebooks
+)/
+'''
+
+[tool.ruff]
+extend-exclude = [
+    "tests/output",
+    "tests/**/output",
+    "tests/**/__snapshots__",
+    "examples/",
+    "docs/",
+    "notebooks/"
+]
+force-exclude = true
+line-length = 120
+extend-ignore = ["E203"]
+select = [
+  "E",  # pycodestyle errors
+  "F",  # Pyflakes
+  "I",  # isort
+]
+# Assume Python 3.8
+target-version = "py38"
+
+[tool.ruff.per-file-ignores]
+# These templates can have long lines
+"linkml/generators/sqlalchemy/sqlalchemy_declarative_template.py" = ["E501"]
+"linkml/generators/sqlalchemy/sqlalchemy_imperative_template.py" = ["E501"]
+
+# Notebooks can have unsorted imports
+"tests/test_notebooks/input/*" = ["E402"]
+
+
+[tool.ruff.mccabe]
+# Unlike Flake8, default to a complexity level of 10.
+max-complexity = 10
+
+
+[tool.codespell]
+# TODO: bring in tests in too
+skip = '.git,*.pdf,*.svg,./tests,pyproject.toml,*.dill,poetry.lock,*.ipynb'
+# Ignore table where words could be split across rows
+# Ignore shortcut specifications like [Ff]alse
+ignore-regex = '(\|.*\|.*\|.*\||\[[A-Z][a-z]\][a-z][a-z])'
+ignore-words-list = 'mater,connexion,infarction'
+count = ""
+quiet-level = 3
+
+[build-system]
+requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
+build-backend = "poetry_dynamic_versioning.backend"

linkml_store-0.0.0/src/linkml_store/__init__.py

@@ -0,0 +1,7 @@
+from pathlib import Path
+
+from linkml_store.api import Client
+
+THIS_DIR = Path(__file__).parent
+
+__all__ = ["Client"]

linkml_store-0.0.0/src/linkml_store/api/__init__.py

@@ -0,0 +1,8 @@
+# flake8: noqa: E402
+from linkml_store.api.collection import Collection
+from linkml_store.api.database import Database
+from linkml_store.api.metadata import MetaData
+from linkml_store.api.client import Client
+# flake8: noqa
+
+__all__ = ["Client", "Database", "MetaData", "Collection"]

linkml_store-0.0.0/src/linkml_store/api/client.py

@@ -0,0 +1,151 @@
+from dataclasses import dataclass
+from typing import Dict, Optional
+
+from linkml_runtime import SchemaView
+
+from linkml_store.api import Database
+from linkml_store.api.stores.duckdb.duckdb_database import DuckDBDatabase
+
+HANDLE_MAP = {
+    "duckdb": DuckDBDatabase,
+}
+
+
+@dataclass
+class Client:
+    """
+    A client provides access to named collections.
+
+    Examples
+    --------
+    >>> client = Client()
+    >>> db = client.attach_database("duckdb", alias="test")
+    >>> collection = db.create_collection("Person")
+    >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
+    >>> collection.add(objs)
+    >>> qr = collection.find()
+    >>> len(qr.rows)
+    2
+    >>> qr.rows[0]["id"]
+    'P1'
+    >>> qr.rows[1]["name"]
+    'Alice'
+    >>> qr = collection.find({"name": "John"})
+    >>> len(qr.rows)
+    1
+    >>> qr.rows[0]["name"]
+    'John'
+
+    """
+
+    handle: Optional[str] = None
+    _databases: Optional[Dict[str, Database]] = None
+
+    def attach_database(
+        self,
+        handle: str,
+        alias: Optional[str] = None,
+        schema_view: Optional[SchemaView] = None,
+        recreate_if_exists=False,
+        **kwargs,
+    ) -> Database:
+        """
+        Associate a database with a handle.
+
+        Examples
+        --------
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="memory")
+        >>> "memory" in client.databases
+        True
+        >>> db = client.attach_database("duckdb:///tmp/another.db", alias="disk")
+        >>> len(client.databases)
+        2
+        >>> "disk" in client.databases
+        True
+
+        :param handle: handle for the database, e.g. duckdb:///foo.db
+        :param alias: alias for the database, e.g foo
+        :param schema_view: schema view to associate with the database
+        :param kwargs:
+        :return:
+
+        """
+        if ":" not in handle:
+            scheme = handle
+            handle = None
+        else:
+            scheme, _ = handle.split(":", 1)
+        if scheme not in HANDLE_MAP:
+            raise ValueError(f"Unknown scheme: {scheme}")
+        cls = HANDLE_MAP[scheme]
+        db = cls(handle=handle, recreate_if_exists=recreate_if_exists, **kwargs)
+        if schema_view:
+            db.set_schema_view(schema_view)
+        if not alias:
+            alias = handle
+        if not self._databases:
+            self._databases = {}
+        self._databases[alias] = db
+        return db
+
+    def get_database(self, name: Optional[str] = None, create_if_not_exists=True, **kwargs) -> Database:
+        """
+        Get a named database.
+
+        Examples
+        --------
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb:///test.db", alias="test")
+        >>> retrieved_db = client.get_database("test")
+        >>> db == retrieved_db
+        True
+
+        :param name:
+        :param create_if_not_exists:
+        :param kwargs:
+        :return:
+
+        """
+        if not name:
+            if not self._databases:
+                raise ValueError("No databases attached and no name provided")
+            if len(self._databases) > 1:
+                raise ValueError("Ambiguous: No name provided and multiple databases attached")
+            return list(self._databases.values())[0]
+        if not self._databases:
+            self._databases = {}
+        if name not in self._databases:
+            if create_if_not_exists:
+                self.attach_database(name, **kwargs)
+            else:
+                raise ValueError(f"Database {name} does not exist")
+        return self._databases[name]
+
+    @property
+    def databases(self) -> Dict[str, Database]:
+        """
+        Return all attached databases
+
+        Examples
+        --------
+        >>> client = Client()
+        >>> _ = client.attach_database("duckdb", alias="test1")
+        >>> _ = client.attach_database("duckdb", alias="test2")
+        >>> len(client.databases)
+        2
+        >>> "test1" in client.databases
+        True
+        >>> "test2" in client.databases
+        True
+        >>> client.databases["test1"].handle
+        'duckdb:///:memory:'
+        >>> client.databases["test2"].handle
+        'duckdb:///:memory:'
+
+        :return:
+
+        """
+        if not self._databases:
+            self._databases = {}
+        return self._databases

linkml_store-0.0.0/src/linkml_store/api/collection.py

@@ -0,0 +1,327 @@
+import logging
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, TextIO, Type, Union
+
+import numpy as np
+from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition
+from linkml_runtime.linkml_model.meta import ArrayExpression
+from pydantic import BaseModel
+
+from linkml_store.api.queries import Query, QueryResult
+from linkml_store.index.index import Index
+
+if TYPE_CHECKING:
+    from linkml_store.api.database import Database
+
+logger = logging.getLogger(__name__)
+
+OBJECT = Union[Dict[str, Any], BaseModel, Type]
+
+IDENTIFIER = str
+FIELD_NAME = str
+
+
+@dataclass
+class Collection:
+    """
+    A collection is an organized set of objects of the same or similar type.
+
+    - For relational databases, a collection is typically a table
+    - For document databases such as MongoDB, a collection is the native type
+    - For a file system, a collection could be a single tabular file such as Parquet or CSV
+    """
+
+    name: str
+    parent: Optional["Database"] = None
+    _indexes: Optional[Dict[str, Index]] = None
+    hidden: Optional[bool] = False
+
+    def add(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
+        """
+        Add one or more objects to the collection
+
+        :param objs:
+        :param kwargs:
+        :return:
+        """
+        raise NotImplementedError
+
+    def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> int:
+        """
+        Delete one or more objects from the collection
+
+        :param objs:
+        :param kwargs:
+        :return:
+        """
+        raise NotImplementedError
+
+    def delete_where(self, where: Optional[Dict[str, Any]] = None, **kwargs) -> int:
+        """
+        Delete objects that match a query
+
+        :param where:
+        :param kwargs:
+        :return:
+        """
+        raise NotImplementedError
+
+    def update(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
+        """
+        Update one or more objects in the collection
+
+        :param objs:
+        :param kwargs:
+        :return:
+        """
+        raise NotImplementedError
+
+    def _create_query(self, **kwargs) -> Query:
+        return Query(from_table=self.name, **kwargs)
+
+    def query(self, query: Query, **kwargs) -> QueryResult:
+        """
+        Run a query against the collection
+
+        :param query:
+        :param kwargs:
+        :return:
+        """
+        return self.parent.query(query, **kwargs)
+
+    def query_facets(self, where: Optional[Dict] = None, facet_columns: List[str] = None) -> Dict[str, Dict[str, int]]:
+        """
+        Run a query to get facet counts for one or more columns.
+
+        This function takes a database connection, a Query object, and a list of column names.
+        It generates and executes a facet count query for each specified column and returns
+        the results as a dictionary where the keys are the column names and the values are
+        pandas DataFrames containing the facet counts.
+
+        The facet count query is generated by modifying the original query's WHERE clause
+        to exclude conditions directly related to the facet column. This allows for counting
+        the occurrences of each unique value in the facet column while still applying the
+        other filtering conditions.
+
+        :param con: A DuckDB database connection.
+        :param query: A Query object representing the base query.
+        :param facet_columns: A list of column names to get facet counts for.
+        :return: A dictionary where keys are column names and values are pandas DataFrames
+                 containing the facet counts for each unique value in the respective column.
+        """
+        raise NotImplementedError
+
+    def get(self, ids: Optional[IDENTIFIER], **kwargs) -> QueryResult:
+        id_field = self.identifier_field
+        q = self._create_query(where_clause={id_field: ids})
+        return self.query(q, **kwargs)
+
+    def find(self, where: Optional[Any] = None, **kwargs) -> QueryResult:
+        query = self._create_query(where_clause=where)
+        return self.query(query, **kwargs)
+
+    def search(
+        self,
+        query: str,
+        where: Optional[Any] = None,
+        index_name: Optional[str] = None,
+        limit: Optional[int] = None,
+        **kwargs,
+    ) -> QueryResult:
+        """
+        Search the collection using a full-text search index.
+
+        :param query:
+        :param where:
+        :param index_name:
+        :param limit:
+        :param kwargs:
+        :return:
+        """
+        if index_name is None:
+            if len(self._indexes) == 1:
+                index_name = list(self._indexes.keys())[0]
+            else:
+                raise ValueError("Multiple indexes found. Please specify an index name.")
+        ix_coll = self.parent.get_collection(self._index_collection_name(index_name))
+        ix = self._indexes.get(index_name)
+        if not ix:
+            raise ValueError(f"No index named {index_name}")
+        qr = ix_coll.find(where=where, limit=-1, **kwargs)
+        index_col = ix.index_field
+        vector_pairs = [(row, np.array(row[index_col], dtype=float)) for row in qr.rows]
+        results = ix.search(query, vector_pairs, limit=limit)
+        new_qr = QueryResult(num_rows=len(results))
+        new_qr.ranked_rows = results
+        return new_qr
+
+    def attach_index(self, index: Index, auto_index=True, **kwargs):
+        """
+        Attach an index to the collection.
+
+        :param index:
+        :param auto_index:
+        :param kwargs:
+        :return:
+        """
+        index_name = index.name
+        if not index_name:
+            raise ValueError("Index must have a name")
+        if not self._indexes:
+            self._indexes = {}
+        self._indexes[index_name] = index
+        if auto_index:
+            all_objs = self.find(limit=-1).rows
+            self.index_objects(all_objs, index_name, **kwargs)
+
+    def _index_collection_name(self, index_name: str) -> str:
+        return f"index__{self.name}_{index_name}"
+
+    def index_objects(self, objs: List[OBJECT], index_name: str, **kwargs):
+        """
+        Index a list of objects
+
+        :param objs:
+        :param index_name:
+        :param kwargs:
+        :return:
+        """
+        ix = self._indexes.get(index_name)
+        if not ix:
+            raise ValueError(f"No index named {index_name}")
+        ix_coll = self.parent.get_collection(self._index_collection_name(index_name), create_if_not_exists=True)
+        vectors = [list(float(e) for e in v) for v in ix.objects_to_vectors(objs)]
+        objects_with_ix = []
+        index_col = ix.index_field
+        for obj, vector in zip(objs, vectors):
+            # TODO: id field
+            objects_with_ix.append({**obj, **{index_col: vector}})
+        ix_coll.add(objects_with_ix, **kwargs)
+
+    def peek(self, limit: Optional[int] = None) -> QueryResult:
+        q = self._create_query()
+        return self.query(q, limit=limit)
+
+    def class_definition(self) -> Optional[ClassDefinition]:
+        """
+        Return the class definition for the collection.
+
+        :return:
+        """
+        sv = self.parent.schema_view
+        if sv:
+            return sv.get_class(self.name)
+        return None
+
+    def identifier_attribute_name(self) -> Optional[str]:
+        """
+        Return the name of the identifier attribute for the collection.
+
+        :return: The name of the identifier attribute, if one exists.
+        """
+        cd = self.class_definition()
+        if cd:
+            for att in cd.attributes.values():
+                if att.identifier:
+                    return att.name
+        return None
+
+    def induce_class_definition_from_objects(self, objs: List[OBJECT], max_sample_size=10) -> ClassDefinition:
+        """
+        Induce a class definition from a list of objects.
+
+        This uses a heuristic procedure to infer the class definition from a list of objects.
+        In general it is recommended you explicitly provide a schema.
+
+        :param objs:
+        :param max_sample_size:
+        :return:
+        """
+        cd = ClassDefinition(self.name)
+        keys = defaultdict(list)
+        for obj in objs[0:max_sample_size]:
+            if isinstance(obj, BaseModel):
+                obj = obj.model_dump()
+            if not isinstance(obj, dict):
+                logger.warning(f"Skipping non-dict object: {obj}")
+                continue
+            for k, v in obj.items():
+                keys[k].append(v)
+        for k, vs in keys.items():
+            multivalueds = []
+            inlineds = []
+            rngs = []
+            exact_dimensions_list = []
+            for v in vs:
+                if v is None:
+                    continue
+                if isinstance(v, np.ndarray):
+                    rngs.append("float")
+                    exact_dimensions_list.append(v.shape)
+                    break
+                if isinstance(v, list):
+                    v = v[0]
+                    multivalueds.append(True)
+                elif isinstance(v, dict):
+                    v = list(v.values())[0]
+                    multivalueds.append(True)
+                else:
+                    multivalueds.append(False)
+                if not v:
+                    continue
+                if isinstance(v, str):
+                    rng = "string"
+                elif isinstance(v, bool):
+                    rng = "boolean"
+                elif isinstance(v, int):
+                    rng = "integer"
+                elif isinstance(v, float):
+                    rng = "float"
+                elif isinstance(v, dict):
+                    rng = None
+                    inlineds.append(True)
+                else:
+                    # raise ValueError(f"No mappings for {type(v)} // v={v}")
+                    rng = None
+                    inlineds.append(False)
+                rngs.append(rng)
+            multivalued = any(multivalueds)
+            inlined = any(inlineds)
+            if multivalued and False in multivalueds:
+                raise ValueError(f"Mixed list non list: {vs} // inferred= {multivalueds}")
+            # if not rngs:
+            #    raise AssertionError(f"Empty rngs for {k} = {vs}")
+            rng = rngs[0] if rngs else None
+            for other_rng in rngs:
+                if rng != other_rng:
+                    raise ValueError(f"Conflict: {rng} != {other_rng} for {vs}")
+            cd.attributes[k] = SlotDefinition(k, range=rng, multivalued=multivalued, inlined=inlined)
+            if exact_dimensions_list:
+                array_expr = ArrayExpression(exact_number_dimensions=len(exact_dimensions_list[0]))
+                cd.attributes[k].array = array_expr
+        sv = self.parent.schema_view
+        sv.schema.classes[self.name] = cd
+        sv.set_modified()
+        return cd
+
+    def import_data(self, location: Union[Path, str, TextIO], **kwargs):
+        """
+        Import data from a file or stream
+
+        :param location:
+        :param kwargs:
+        :return:
+        """
+        raise NotImplementedError
+
+    def export_data(self, location: Union[Path, str, TextIO], **kwargs):
+        """
+        Export data to a file or stream
+
+        :param location:
+        :param kwargs:
+        :return:
+        """
+        raise NotImplementedError