linkml-store 0.1.9__py3-none-any.whl → 0.1.10__py3-none-any.whl

linkml_store/api/client.py

@@ -98,7 +98,7 @@ class Client:
         """
         return self.metadata.base_dir
 
-    def from_config(self, config: Union[ClientConfig, str, Path], base_dir=None, **kwargs):
+    def from_config(self, config: Union[ClientConfig, dict, str, Path], base_dir=None, **kwargs):
         """
         Create a client from a configuration.
 
@@ -118,11 +118,13 @@ class Client:
         :return:
 
         """
+        if isinstance(config, dict):
+            config = ClientConfig(**config)
         if isinstance(config, Path):
             config = str(config)
         if isinstance(config, str):
-            if not base_dir:
-                base_dir = Path(config).parent
+            # if not base_dir:
+            #    base_dir = Path(config).parent
             parsed_obj = yaml.safe_load(open(config))
             config = ClientConfig(**parsed_obj)
         self.metadata = config
@@ -133,8 +135,15 @@ class Client:
 
     def _initialize_databases(self, **kwargs):
         for name, db_config in self.metadata.databases.items():
-            handle = db_config.handle.format(base_dir=self.base_dir)
+            base_dir = self.base_dir
+            logger.info(f"Initializing database: {name}, base_dir: {base_dir}")
+            if not base_dir:
+                base_dir = Path.cwd()
+                logger.info(f"Using current working directory: {base_dir}")
+            handle = db_config.handle.format(base_dir=base_dir)
             db_config.handle = handle
+            if db_config.schema_location:
+                db_config.schema_location = db_config.schema_location.format(base_dir=base_dir)
             db = self.attach_database(handle, alias=name, **kwargs)
             db.from_config(db_config)
 

linkml_store/api/collection.py

@@ -4,7 +4,7 @@ import hashlib
 import logging
 from collections import defaultdict
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Generic, Iterator, List, Optional, TextIO, Tuple, Type, Union
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, Generic, Iterator, List, Optional, TextIO, Tuple, Type, Union
 
 import numpy as np
 from linkml_runtime import SchemaView
@@ -64,6 +64,7 @@ class Collection(Generic[DatabaseType]):
     # hidden: Optional[bool] = False
 
     metadata: Optional[CollectionConfig] = None
+    default_index_name: ClassVar[str] = "simple"
 
     def __init__(
         self, name: str, parent: Optional["Database"] = None, metadata: Optional[CollectionConfig] = None, **kwargs
@@ -421,7 +422,30 @@ class Collection(Generic[DatabaseType]):
         **kwargs,
     ) -> QueryResult:
         """
-        Search the collection using a full-text search index.
+        Search the collection using a text-based index index.
+
+        Example:
+
+        >>> from linkml_store import Client
+        >>> from linkml_store.utils.format_utils import load_objects
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb")
+        >>> collection = db.create_collection("Country")
+        >>> objs = load_objects("tests/input/countries/countries.jsonl")
+        >>> collection.insert(objs)
+
+        Now let's index, using the simple trigram-based index
+
+        >>> index = get_indexer("simple")
+        >>> collection.attach_indexer(index)
+
+        Now let's find all objects:
+
+        >>> qr = collection.search("France")
+        >>> score, top_obj = qr.ranked_rows[0]
+        >>> assert score > 0.1
+        >>> top_obj["code"]
+        'FR'
 
         :param query:
         :param where:
@@ -431,12 +455,18 @@ class Collection(Generic[DatabaseType]):
         :return:
         """
         if index_name is None:
-            if len(self._indexers) == 1:
-                index_name = list(self._indexers.keys())[0]
+            if len(self.indexers) == 1:
+                index_name = list(self.indexers.keys())[0]
             else:
-                raise ValueError("Multiple indexes found. Please specify an index name.")
+                logger.warning("Multiple indexes found. Using default index.")
+                index_name = self.default_index_name
         ix_coll = self.parent.get_collection(self._index_collection_name(index_name))
-        ix = self._indexers.get(index_name)
+        if index_name not in self.indexers:
+            ix = get_indexer(index_name)
+            if not self._indexers:
+                self._indexers = {}
+            self._indexers[index_name] = ix
+        ix = self.indexers.get(index_name)
         if not ix:
             raise ValueError(f"No index named {index_name}")
         qr = ix_coll.find(where=where, limit=-1, **kwargs)
@@ -453,7 +483,10 @@ class Collection(Generic[DatabaseType]):
     @property
     def is_internal(self) -> bool:
         """
-        Check if the collection is internal
+        Check if the collection is internal.
+
+        Internal collections are hidden by default. Examples of internal collections
+        include shadow "index" collections
 
         :return:
         """
@@ -469,6 +502,45 @@ class Collection(Generic[DatabaseType]):
         """
         Attach an index to the collection.
 
+        As an example, first let's create a collection in a database:
+
+        >>> from linkml_store import Client
+        >>> from linkml_store.utils.format_utils import load_objects
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb")
+        >>> collection = db.create_collection("Country")
+        >>> objs = load_objects("tests/input/countries/countries.jsonl")
+        >>> collection.insert(objs)
+
+        We will create two indexes - one that indexes the whole object
+        (default behavior), the other one indexes the name only
+
+        >>> full_index = get_indexer("simple")
+        >>> full_index.name = "full"
+        >>> name_index = get_indexer("simple", text_template="{name}")
+        >>> name_index.name = "name"
+        >>> collection.attach_indexer(full_index)
+        >>> collection.attach_indexer(name_index)
+
+        Now let's find objects using the full index, using the string "France".
+        We expect the country France to be the top hit, but the score will
+        be less than zero because we did not match all fields in the object.
+
+        >>> qr = collection.search("France", index_name="full")
+        >>> score, top_obj = qr.ranked_rows[0]
+        >>> assert score > 0.1
+        >>> assert score < 0.5
+        >>> top_obj["code"]
+        'FR'
+
+        Now using the name index
+
+        >>> qr = collection.search("France", index_name="name")
+        >>> score, top_obj = qr.ranked_rows[0]
+        >>> assert score > 0.99
+        >>> top_obj["code"]
+        'FR'
+
         :param index:
         :param name:
         :param auto_index: Automatically index all objects in the collection
@@ -504,15 +576,18 @@ class Collection(Generic[DatabaseType]):
 
     def index_objects(self, objs: List[OBJECT], index_name: str, replace=False, **kwargs):
         """
-        Index a list of objects
+        Index a list of objects using a specified index.
+
+        By default, the indexed objects will be stored in a shadow
+        collection in the same database, with additional fields for the index vector
 
         :param objs:
-        :param index_name:
+        :param index_name: e.g. simple, llm
         :param replace:
         :param kwargs:
         :return:
         """
-        ix = self._indexers.get(index_name)
+        ix = self._indexers.get(index_name, None)
         if not ix:
             raise ValueError(f"No index named {index_name}")
         ix_coll_name = self._index_collection_name(index_name)

linkml_store/api/config.py

@@ -3,7 +3,11 @@ from typing import Any, Dict, List, Optional
 from pydantic import BaseModel, Field
 
 
-class CollectionConfig(BaseModel):
+class ConfiguredBaseModel(BaseModel, extra="forbid"):
+    pass
+
+
+class CollectionConfig(ConfiguredBaseModel):
     name: Optional[str] = Field(
         default=None,
         description="An optional name for the collection",
@@ -42,7 +46,7 @@ class CollectionConfig(BaseModel):
     )
 
 
-class DatabaseConfig(BaseModel):
+class DatabaseConfig(ConfiguredBaseModel):
     handle: str = Field(
         default="duckdb:///:memory:",
         description="The database handle, e.g., 'duckdb:///:memory:' or 'mongodb://localhost:27017'",
@@ -86,7 +90,7 @@ class DatabaseConfig(BaseModel):
     )
 
 
-class ClientConfig(BaseModel):
+class ClientConfig(ConfiguredBaseModel):
     handle: Optional[str] = Field(
         default=None,
         description="The client handle",
@@ -95,6 +99,10 @@ class ClientConfig(BaseModel):
         default={},
         description="A dictionary of database configurations",
     )
+    default_database: Optional[str] = Field(
+        default=None,
+        description="The default database",
+    )
     schema_path: Optional[str] = Field(
         default=None,
         description="The path to the LinkML schema file",

linkml_store/api/stores/duckdb/duckdb_collection.py

@@ -90,7 +90,9 @@ class DuckDBCollection(Collection):
         cd = self.class_definition()
         with self.parent.engine.connect() as conn:
             if not facet_columns:
-                facet_columns = list(self.class_definition().attributes.keys())
+                if not cd:
+                    raise ValueError(f"No class definition found for {self.target_class_name}")
+                facet_columns = list(cd.attributes.keys())
             for col in facet_columns:
                 logger.debug(f"Faceting on {col}")
                 if isinstance(col, tuple):
@@ -101,7 +103,7 @@ class DuckDBCollection(Collection):
                 facet_query_str = facet_count_sql(facet_query, col, multivalued=sd.multivalued)
                 logger.debug(f"Facet query: {facet_query_str}")
                 rows = list(conn.execute(text(facet_query_str)))
-                results[col] = rows
+                results[col] = [tuple(row) for row in rows]
             return results
 
     def _sqla_table(self, cd: ClassDefinition) -> Table:
@@ -110,7 +112,7 @@ class DuckDBCollection(Collection):
         cols = []
         for att in schema_view.class_induced_slots(cd.name):
             typ = TMAP.get(att.range, sqla.String)
-            if att.inlined:
+            if att.inlined or att.inlined_as_list:
                 typ = sqla.JSON
             if att.multivalued:
                 typ = sqla.ARRAY(typ, dimensions=1)

linkml_store/api/stores/duckdb/duckdb_database.py

@@ -31,6 +31,18 @@ logger = logging.getLogger(__name__)
 
 
 class DuckDBDatabase(Database):
+    """
+    An adapter for DuckDB databases.
+
+    Note that this adapter does not make use of a LinkML relational model transformation and
+    SQL Alchemy ORM layer. Instead, it attempts to map each collection (which is of type
+    some LinkML class) to a *single* DuckDB table. New tables are not created for nested references,
+    and linking tables are not created for many-to-many relationships.
+
+    Instead the native DuckDB ARRAY type is used to store multivalued attributes, and DuckDB JSON
+    types are used for nested inlined objects.
+    """
+
     _connection: DuckDBPyConnection = None
     _engine: sqlalchemy.Engine = None
     collection_class = DuckDBCollection
@@ -103,7 +115,14 @@ class DuckDBDatabase(Database):
                         if row[col]:
                             if isinstance(row[col], list):
                                 for i in range(len(row[col])):
-                                    row[col][i] = json.loads(row[col][i])
+                                    try:
+                                        parsed_val = json.loads(row[col][i])
+                                    except json.JSONDecodeError as e:
+                                        logger.error(f"Failed to parse col {col}[{i}] == {row[col][i]}")
+                                        raise e
+                                    row[col][i] = parsed_val
+                            elif isinstance(row[col], dict):
+                                pass
                             else:
                                 row[col] = json.loads(row[col])
             qr.set_rows(pd.DataFrame(rows))

linkml_store/api/stores/filesystem/filesystem_collection.py

@@ -170,6 +170,8 @@ class FileSystemCollection(Collection[DatabaseType]):
             for fc in facet_columns:
                 if fc in row:
                     v = row[fc]
+                    if not isinstance(v, str):
+                        v = str(v)
                     if v not in facet_results[fc]:
                         facet_results[fc][v] = 1
                     else:

linkml_store/api/stores/mongodb/mongodb_collection.py

@@ -2,7 +2,6 @@ import logging
 from copy import copy
 from typing import Any, Dict, List, Optional, Tuple, Union
 
-from linkml_runtime.linkml_model import SlotDefinition
 from pymongo.collection import Collection as MongoCollection
 
 from linkml_store.api import Collection
@@ -38,13 +37,15 @@ class MongoDBCollection(Collection):
             del obj["_id"]
         self._post_insert_hook(objs)
 
-    def query(self, query: Query, limit: Optional[int] = None, **kwargs) -> QueryResult:
+    def query(self, query: Query, limit: Optional[int] = None, offset: Optional[int] = None, **kwargs) -> QueryResult:
         mongo_filter = self._build_mongo_filter(query.where_clause)
         limit = limit or query.limit
+        cursor = self.mongo_collection.find(mongo_filter)
         if limit and limit >= 0:
-            cursor = self.mongo_collection.find(mongo_filter).limit(limit)
-        else:
-            cursor = self.mongo_collection.find(mongo_filter)
+            cursor = cursor.limit(limit)
+        offset = offset or query.offset
+        if offset and offset >= 0:
+            cursor = cursor.skip(offset)
 
         def _as_row(row: dict):
             row = copy(row)
@@ -63,46 +64,87 @@ class MongoDBCollection(Collection):
                 mongo_filter[field] = value
         return mongo_filter
 
+    from typing import Any, Dict, List, Union
+
     def query_facets(
-        self, where: Dict = None, facet_columns: List[str] = None, facet_limit=DEFAULT_FACET_LIMIT, **kwargs
-    ) -> Dict[str, List[Tuple[Any, int]]]:
+        self,
+        where: Dict = None,
+        facet_columns: List[Union[str, Tuple[str, ...]]] = None,
+        facet_limit=DEFAULT_FACET_LIMIT,
+        **kwargs,
+    ) -> Dict[Union[str, Tuple[str, ...]], List[Tuple[Any, int]]]:
         results = {}
-        cd = self.class_definition()
         if not facet_columns:
             facet_columns = list(self.class_definition().attributes.keys())
 
         for col in facet_columns:
             logger.debug(f"Faceting on {col}")
+
+            # Handle tuple columns
+            if isinstance(col, tuple):
+                group_id = {k.replace(".", "_"): f"${k}" for k in col}
+                all_fields = col
+            else:
+                group_id = f"${col}"
+                all_fields = [col]
+
+            # Initial pipeline without unwinding
+            facet_pipeline = [
+                {"$match": where} if where else {"$match": {}},
+                {"$group": {"_id": group_id, "count": {"$sum": 1}}},
+                {"$sort": {"count": -1}},
+                {"$limit": facet_limit},
+            ]
+
+            logger.info(f"Initial facet pipeline: {facet_pipeline}")
+            initial_results = list(self.mongo_collection.aggregate(facet_pipeline))
+
+            # Check if we need to unwind based on the results
+            needs_unwinding = False
             if isinstance(col, tuple):
-                sd = SlotDefinition(name="PLACEHOLDER")
+                needs_unwinding = any(
+                    isinstance(result["_id"], dict) and any(isinstance(v, list) for v in result["_id"].values())
+                    for result in initial_results
+                )
             else:
-                if col in cd.attributes:
-                    sd = cd.attributes[col]
-                else:
-                    logger.info(f"No schema metadata for {col}")
-                    sd = SlotDefinition(name=col)
-            group = {"$group": {"_id": f"${col}", "count": {"$sum": 1}}}
+                needs_unwinding = any(isinstance(result["_id"], list) for result in initial_results)
+
+            if needs_unwinding:
+                logger.info(f"Detected array values for {col}, unwinding...")
+                facet_pipeline = [{"$match": where} if where else {"$match": {}}]
+
+                # Unwind each field if needed
+                for field in all_fields:
+                    field_parts = field.split(".")
+                    for i in range(len(field_parts)):
+                        facet_pipeline.append({"$unwind": f"${'.'.join(field_parts[:i + 1])}"})
+
+                facet_pipeline.extend(
+                    [
+                        {"$group": {"_id": group_id, "count": {"$sum": 1}}},
+                        {"$sort": {"count": -1}},
+                        {"$limit": facet_limit},
+                    ]
+                )
+
+                logger.info(f"Updated facet pipeline with unwinding: {facet_pipeline}")
+                facet_results = list(self.mongo_collection.aggregate(facet_pipeline))
+            else:
+                facet_results = initial_results
+
+            logger.info(f"Facet results: {facet_results}")
+
+            # Process results
             if isinstance(col, tuple):
-                q = {k.replace(".", ""): f"${k}" for k in col}
-                group["$group"]["_id"] = q
-            if sd and sd.multivalued:
-                facet_pipeline = [
-                    {"$match": where} if where else {"$match": {}},
-                    {"$unwind": f"${col}"},
-                    group,
-                    {"$sort": {"count": -1}},
-                    {"$limit": facet_limit},
+                results[col] = [
+                    (tuple(result["_id"].values()), result["count"])
+                    for result in facet_results
+                    if result["_id"] is not None and all(v is not None for v in result["_id"].values())
                 ]
             else:
-                facet_pipeline = [
-                    {"$match": where} if where else {"$match": {}},
-                    group,
-                    {"$sort": {"count": -1}},
-                    {"$limit": facet_limit},
+                results[col] = [
+                    (result["_id"], result["count"]) for result in facet_results if result["_id"] is not None
                 ]
-            logger.info(f"Facet pipeline: {facet_pipeline}")
-            facet_results = list(self.mongo_collection.aggregate(facet_pipeline))
-            results[col] = [(result["_id"], result["count"]) for result in facet_results]
 
         return results
 

linkml_store/cli.py

@@ -159,7 +159,15 @@ def cli(ctx, verbose: int, quiet: bool, stacktrace: bool, database, collection,
 @click.option("--object", "-i", multiple=True, help="Input object as YAML")
 @click.pass_context
 def insert(ctx, files, object, format):
-    """Insert objects from files (JSON, YAML, TSV) into the specified collection."""
+    """Insert objects from files (JSON, YAML, TSV) into the specified collection.
+
+    Using a configuration:
+
+        linkml-store -C config.yaml -c genes insert data/genes/*.json
+
+    Note: if you don't provide a schema this will be inferred, but it is
+    usually better to provide an explicit schema
+    """
     settings = ctx.obj["settings"]
     collection = settings.collection
     if not collection:

linkml_store/index/__init__.py

@@ -22,7 +22,7 @@ def get_indexer_class(name: str) -> Type[Indexer]:
     return INDEXER_CLASSES[name]
 
 
-def get_indexer(name: str, **kwargs) -> Indexer:
+def get_indexer(index_type: str, **kwargs) -> Indexer:
     """
     Get an indexer by name.
 
@@ -31,7 +31,9 @@ def get_indexer(name: str, **kwargs) -> Indexer:
     :return: the indexer
     """
     kwargs = {k: v for k, v in kwargs.items() if v is not None}
-    cls = get_indexer_class(name)
-    kwargs["name"] = name
+    cls = get_indexer_class(index_type)
+    kwargs["index_type"] = index_type
     indexer = cls(**kwargs)
+    if not indexer.name:
+        indexer.name = index_type
     return indexer

linkml_store/index/indexer.py

@@ -28,6 +28,7 @@ class Indexer(BaseModel):
     """
 
     name: Optional[str] = None
+    index_type: Optional[str] = None
     index_function: Optional[Callable] = None
     distance_function: Optional[Callable] = None
     index_attributes: Optional[List[str]] = None
@@ -93,13 +94,17 @@ class Indexer(BaseModel):
                 if "{%" in self.text_template or "{{" in self.text_template:
                     logger.info("Detected Jinja2 syntax in text template")
                     syntax = TemplateSyntaxEnum.jinja2
-            if syntax and syntax == TemplateSyntaxEnum.jinja2:
+            if not syntax:
+                syntax = TemplateSyntaxEnum.fstring
+            if syntax == TemplateSyntaxEnum.jinja2:
                 from jinja2 import Template
 
                 template = Template(self.text_template)
                 return template.render(**obj)
-            else:
+            elif syntax == TemplateSyntaxEnum.fstring:
                 return self.text_template.format(**obj)
+            else:
+                raise NotImplementedError(f"Cannot handle template syntax: {syntax}")
         return str(obj)
 
     def search(

linkml_store/utils/format_utils.py

@@ -64,7 +64,7 @@ def load_objects(
     elif format == Format.JSONL or (not format and file_path.endswith(".jsonl")):
         objs = [json.loads(line) for line in f]
     elif format == Format.YAML or (not format and (file_path.endswith(".yaml") or file_path.endswith(".yml"))):
-        if expected_type and expected_type == list:
+        if expected_type and expected_type == list:  # noqa E721
             objs = list(yaml.safe_load_all(f))
         else:
             objs = yaml.safe_load(f)

linkml_store/utils/schema_utils.py

@@ -0,0 +1,23 @@
+from typing import List
+
+from linkml_runtime import SchemaView
+from linkml_runtime.linkml_model import SlotDefinition
+
+
+def path_to_attribute_list(class_name: str, path: str, schema_view: SchemaView) -> List[SlotDefinition]:
+    """
+    Convert a path to a list of attributes.
+
+    :param path:
+    :return:
+    """
+    parts = path.split(".")
+    att_list = []
+    while parts:
+        part = parts.pop(0)
+        att = schema_view.induced_slot(part, class_name)
+        if not att:
+            raise ValueError(f"Attribute {part} not found in class {class_name}")
+        att_list.append(att)
+        class_name = att.range
+    return att_list

linkml_store/webapi/html/__init__.py

@@ -0,0 +1,3 @@
+from pathlib import Path
+
+HTML_TEMPLATES_DIR = Path(__file__).parent

linkml_store/webapi/html/base.html.j2

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{% block title %}LinkML Store API{% endblock %}</title>
+    <style>
+        body { font-family: Arial, sans-serif; line-height: 1.6; padding: 20px; }
+        h1 { color: #333; }
+        a { color: #0066cc; }
+        .navigation { margin-bottom: 20px; }
+        .content { margin-top: 20px; }
+    </style>
+</head>
+<body>
+    <div class="navigation">
+        <a href="/pages/">Home</a> |
+        <a href="/pages/databases">Databases</a>
+    </div>
+    <div class="content">
+        {% block content %}{% endblock %}
+    </div>
+</body>
+</html>

linkml_store/webapi/html/collection_details.html.j2

@@ -0,0 +1,15 @@
+{% extends "base.html.j2" %}
+
+{% block content %}
+<h1>{{ response.meta.title }}</h1>
+<p>Name: {{ params.collection_name }}</p>
+
+<h2>Collections</h2>
+<ul>
+{% for collection in response.data.collections %}
+    <li>
+        <a href="/pages{{ collection.links|selectattr('rel', 'equalto', 'self')|first|attr('href') }}">{{ collection.name }}</a>
+    </li>
+{% endfor %}
+</ul>
+{% endblock %}

linkml_store/webapi/html/database_details.html.j2

@@ -0,0 +1,16 @@
+{% extends "base.html.j2" %}
+
+{% block content %}
+<h1>{{ response.meta.title }}</h1>
+<p>Handle: {{ response.data.handle }}</p>
+<p>Number of collections: {{ response.data.num_collections }}</p>
+
+<h2>Collections</h2>
+<ul>
+{% for collection in response.data.collections %}
+    <li>
+        <a href="/pages{{ collection.links|selectattr('rel', 'equalto', 'self')|first|attr('href') }}">{{ collection.name }}</a>
+    </li>
+{% endfor %}
+</ul>
+{% endblock %}

linkml_store/webapi/html/databases.html.j2

@@ -0,0 +1,14 @@
+{% extends "base.html.j2" %}
+
+{% block title %}LinkML Store API - Databases{% endblock %}
+
+{% block content %}
+<h1>Databases</h1>
+<ul>
+{% for db in response.data.databases %}
+    <li>
+        <a href="/pages/databases/{{ db.name }}">{{ db.name }}</a>
+    </li>
+{% endfor %}
+</ul>
+{% endblock %}