linkml-store 0.1.14__py3-none-any.whl → 0.2.1__py3-none-any.whl

linkml_store/api/collection.py

@@ -226,6 +226,18 @@ class Collection(Generic[DatabaseType]):
         self._initialized = True
         patches = [{"op": "add", "path": "/0", "value": obj} for obj in objs]
         self._broadcast(patches, **kwargs)
+        self._post_modification_hook(**kwargs)
+
+    def _post_delete_hook(self, **kwargs):
+        self._post_modification_hook(**kwargs)
+
+    def _post_modification_hook(self, **kwargs):
+        for indexer in self.indexers.values():
+            ix_collection_name = self.get_index_collection_name(indexer)
+            ix_collection = self.parent.get_collection(ix_collection_name)
+            # Currently updating the source triggers complete reindexing
+            # TODO: make this more efficient by only deleting modified
+            ix_collection.delete_where({})
 
     def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> Optional[int]:
         """
@@ -476,7 +488,7 @@ class Collection(Generic[DatabaseType]):
         Now let's index, using the simple trigram-based index
 
         >>> index = get_indexer("simple")
-        >>> collection.attach_indexer(index)
+        >>> _ = collection.attach_indexer(index)
 
         Now let's find all objects:
 
@@ -514,7 +526,10 @@ class Collection(Generic[DatabaseType]):
         if ix_coll.size() == 0:
             logger.info(f"Index {index_name} is empty; indexing all objects")
             all_objs = self.find(limit=-1).rows
-            self.index_objects(all_objs, index_name, replace=True, **kwargs)
+            if all_objs:
+                # print(f"Index {index_name} is empty; indexing all objects {len(all_objs)}")
+                self.index_objects(all_objs, index_name, replace=True, **kwargs)
+                assert ix_coll.size() > 0
         qr = ix_coll.find(where=where, limit=-1, **kwargs)
         index_col = ix.index_field
         # TODO: optimize this for large indexes
@@ -648,7 +663,31 @@ class Collection(Generic[DatabaseType]):
         """
         return self.find({}, limit=1).num_rows
 
-    def attach_indexer(self, index: Union[Indexer, str], name: Optional[str] = None, auto_index=True, **kwargs):
+    def rows_iter(self) -> Iterable[OBJECT]:
+        """
+        Return an iterator over the objects in the collection.
+
+        :return:
+        """
+        yield from self.find({}, limit=-1).rows
+
+    def rows(self) -> List[OBJECT]:
+        """
+        Return a list of objects in the collection.
+
+        :return:
+        """
+        return list(self.rows_iter())
+
+    def ranked_rows(self) -> List[Tuple[float, OBJECT]]:
+        """
+        Return a list of objects in the collection, with scores.
+        """
+        return [(n, obj) for n, obj in enumerate(self.rows_iter())]
+
+    def attach_indexer(
+        self, index: Union[Indexer, str], name: Optional[str] = None, auto_index=True, **kwargs
+    ) -> Indexer:
         """
         Attach an index to the collection.
 
@@ -669,8 +708,8 @@ class Collection(Generic[DatabaseType]):
         >>> full_index.name = "full"
         >>> name_index = get_indexer("simple", text_template="{name}")
         >>> name_index.name = "name"
-        >>> collection.attach_indexer(full_index)
-        >>> collection.attach_indexer(name_index)
+        >>> _ = collection.attach_indexer(full_index)
+        >>> _ = collection.attach_indexer(name_index)
 
         Now let's find objects using the full index, using the string "France".
         We expect the country France to be the top hit, but the score will
@@ -713,6 +752,10 @@ class Collection(Generic[DatabaseType]):
             all_objs = self.find(limit=-1).rows
             logger.info(f"Auto-indexing {len(all_objs)} objects")
             self.index_objects(all_objs, index_name, replace=True, **kwargs)
+        return index
+
+    def get_index_collection_name(self, indexer: Indexer) -> str:
+        return self._index_collection_name(indexer.name)
 
     def _index_collection_name(self, index_name: str) -> str:
         """

linkml_store/api/database.py

@@ -268,7 +268,7 @@ class Database(ABC, Generic[CollectionType]):
         metadata: Optional[CollectionConfig] = None,
         recreate_if_exists=False,
         **kwargs,
-    ) -> CollectionType:
+    ) -> Collection:
         """
         Create a new collection in the current database.
 
@@ -760,6 +760,12 @@ class Database(ABC, Generic[CollectionType]):
         """
         Export a database to a file or location.
 
+        >>> from linkml_store.api.client import Client
+        >>> client = Client()
+        >>> db = client.attach_database("duckdb", alias="test")
+        >>> db.import_database("tests/input/iris.csv", Format.CSV, collection_name="iris")
+        >>> db.export_database("/tmp/iris.yaml", Format.YAML)
+
         :param location: location of the file
         :param target_format: target format
         :param kwargs: additional arguments

linkml_store/api/queries.py

@@ -40,7 +40,9 @@ class FacetCountResult(BaseModel):
 
 class QueryResult(BaseModel):
     """
-    A query result
+    A query result.
+
+    TODO: make this a subclass of Collection
     """
 
     query: Optional[Query] = None

linkml_store/api/stores/duckdb/duckdb_collection.py

@@ -36,6 +36,9 @@ class DuckDBCollection(Collection):
         logger.info(f"Inserting into: {self.alias} // T={table.name}")
         engine = self.parent.engine
         col_names = [c.name for c in table.columns]
+        bad_objs = [obj for obj in objs if not isinstance(obj, dict)]
+        if bad_objs:
+            logger.error(f"Bad objects: {bad_objs}")
         objs = [{k: obj.get(k, None) for k in col_names} for obj in objs]
         with engine.connect() as conn:
             with conn.begin():
@@ -47,8 +50,9 @@ class DuckDBCollection(Collection):
         if not isinstance(objs, list):
             objs = [objs]
         cd = self.class_definition()
-        if not cd:
+        if not cd or not cd.attributes:
             cd = self.induce_class_definition_from_objects(objs)
+        assert cd.attributes
         table = self._sqla_table(cd)
         engine = self.parent.engine
         with engine.connect() as conn:
@@ -58,7 +62,8 @@ class DuckDBCollection(Collection):
                 stmt = stmt.compile(engine)
                 conn.execute(stmt)
                 conn.commit()
-        return
+        self._post_delete_hook()
+        return None
 
     def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> Optional[int]:
         logger.info(f"Deleting from {self.target_class_name} where: {where}")
@@ -84,6 +89,7 @@ class DuckDBCollection(Collection):
             if deleted_rows_count == 0 and not missing_ok:
                 raise ValueError(f"No rows found for {where}")
             conn.commit()
+            self._post_delete_hook()
             return deleted_rows_count if deleted_rows_count > -1 else None
 
     def query_facets(

linkml_store/cli.py

@@ -7,6 +7,7 @@ from typing import Optional
 import click
 import yaml
 from linkml_runtime.dumpers import json_dumper
+from linkml_runtime.utils.formatutils import underscore
 from pydantic import BaseModel
 
 from linkml_store import Client
@@ -17,6 +18,7 @@ from linkml_store.index import get_indexer
 from linkml_store.index.implementations.simple_indexer import SimpleIndexer
 from linkml_store.index.indexer import Indexer
 from linkml_store.inference import get_inference_engine
+from linkml_store.inference.evaluation import evaluate_predictor, score_text_overlap
 from linkml_store.inference.inference_config import InferenceConfig
 from linkml_store.inference.inference_engine import ModelSerialization
 from linkml_store.utils.format_utils import Format, guess_format, load_objects, render_output, write_output
@@ -74,6 +76,8 @@ class ContextSettings(BaseModel):
         if name is None:
             # if len(self.database.list_collections()) > 1:
             #    raise ValueError("Collection must be specified if there are multiple collections.")
+            if not self.database:
+                return None
             if not self.database.list_collections():
                 return None
             name = list(self.database.list_collections())[0]
@@ -130,7 +134,7 @@ def cli(ctx, verbose: int, quiet: bool, stacktrace: bool, database, collection,
         logger.setLevel(logging.ERROR)
     ctx.ensure_object(dict)
     if input:
-        stem = Path(input).stem
+        stem = underscore(Path(input).stem)
         database = "duckdb"
         collection = stem
         config = ClientConfig(databases={"duckdb": {"collections": {stem: {"source": {"local_path": input}}}}})
@@ -216,7 +220,10 @@ def insert(ctx, files, object, format):
 @click.option("--object", "-i", multiple=True, help="Input object as YAML")
 @click.pass_context
 def store(ctx, files, object, format):
-    """Store objects from files (JSON, YAML, TSV) into the specified collection."""
+    """Store objects from files (JSON, YAML, TSV) into the database.
+
+    Note: this is similar to insert, but a collection does not need to be specified
+    """
     settings = ctx.obj["settings"]
     db = settings.database
     if not files and not object:
@@ -496,12 +503,16 @@ def describe(ctx, where, output_type, output, limit):
 @click.option(
     "--predictor-type", "-t", default="sklearn", show_default=True, type=click.STRING, help="Type of predictor"
 )
+@click.option("--evaluation-count", "-n", type=click.INT, help="Number of examples to evaluate over")
+@click.option("--evaluation-match-function", help="Name of function to use for matching objects in eval")
 @click.option("--query", "-q", type=click.STRING, help="query term")
 @click.pass_context
 def infer(
     ctx,
     inference_config_file,
     query,
+    evaluation_count,
+    evaluation_match_function,
     training_test_data_split,
     predictor_type,
     target_attribute,
@@ -545,25 +556,28 @@ def infer(
     else:
         query_obj = None
     collection = ctx.obj["settings"].collection
-    atts = collection.class_definition().attributes.keys()
+    if collection:
+        atts = collection.class_definition().attributes.keys()
+    else:
+        atts = []
+    if feature_attributes:
+        features = feature_attributes.split(",")
+        features = [f.strip() for f in features]
+    else:
+        if query_obj:
+            features = query_obj.keys()
+        else:
+            features = None
+    if target_attribute:
+        target_attributes = list(target_attribute)
+    else:
+        target_attributes = [att for att in atts if att not in features]
     if model_format:
         model_format = ModelSerialization(model_format)
     if load_model:
         predictor = get_inference_engine(predictor_type)
         predictor = type(predictor).load_model(load_model)
     else:
-        if feature_attributes:
-            features = feature_attributes.split(",")
-            features = [f.strip() for f in features]
-        else:
-            if query_obj:
-                features = query_obj.keys()
-            else:
-                features = None
-        if target_attribute:
-            target_attributes = list(target_attribute)
-        else:
-            target_attributes = [att for att in atts if att not in features]
         if inference_config_file:
             config = InferenceConfig.from_file(inference_config_file)
         else:
@@ -571,14 +585,26 @@ def infer(
         if training_test_data_split:
             config.train_test_split = training_test_data_split
         predictor = get_inference_engine(predictor_type, config=config)
-        predictor.load_and_split_data(collection)
+        if collection:
+            predictor.load_and_split_data(collection)
         predictor.initialize_model()
     if export_model:
         logger.info(f"Exporting model to {export_model} in {model_format}")
         predictor.export_model(export_model, model_format)
     if not query_obj:
-        if not export_model:
-            raise ValueError("Query must be specified if not exporting model")
+        if not export_model and not evaluation_count:
+            raise ValueError("Query or evaluate must be specified if not exporting model")
+    if evaluation_count:
+        if evaluation_match_function == "score_text_overlap":
+            match_function_fn = score_text_overlap
+        elif evaluation_match_function is not None:
+            raise ValueError(f"Unknown match function: {evaluation_match_function}")
+        else:
+            match_function_fn = None
+        outcome = evaluate_predictor(
+            predictor, target_attributes, evaluation_count=evaluation_count, match_function=match_function_fn
+        )
+        print(f"Outcome: {outcome} // accuracy: {outcome.accuracy}")
     if query_obj:
         result = predictor.derive(query_obj)
         dumped_obj = result.model_dump(exclude_none=True)

linkml_store/index/implementations/llm_indexer.py

@@ -1,11 +1,13 @@
 import logging
 from pathlib import Path
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, List, Optional
 
 import numpy as np
+from tiktoken import encoding_for_model
 
 from linkml_store.api.config import CollectionConfig
 from linkml_store.index.indexer import INDEX_ITEM, Indexer
+from linkml_store.utils.llm_utils import get_token_limit, render_formatted_text
 
 if TYPE_CHECKING:
     import llm
@@ -29,6 +31,7 @@ class LLMIndexer(Indexer):
     cached_embeddings_database: str = None
     cached_embeddings_collection: str = None
     cache_queries: bool = False
+    truncation_method: Optional[str] = None
 
     @property
     def embedding_model(self):
@@ -62,6 +65,21 @@ class LLMIndexer(Indexer):
         """
         logging.info(f"Converting {len(texts)} texts to vectors")
         model = self.embedding_model
+        token_limit = get_token_limit(model.model_id)
+        encoding = encoding_for_model("gpt-4o")
+
+        def truncate_text(text: str) -> str:
+            # split into tokens every 1000 chars:
+            parts = [text[i : i + 1000] for i in range(0, len(text), 1000)]
+            return render_formatted_text(
+                lambda x: "".join(x),
+                parts,
+                encoding,
+                token_limit,
+            )
+
+        texts = [truncate_text(text) for text in texts]
+
         if self.cached_embeddings_database and (cache is None or cache or self.cache_queries):
             model_id = model.model_id
             if not model_id:
@@ -88,7 +106,7 @@ class LLMIndexer(Indexer):
                 embeddings_collection = embeddings_db.create_collection(coll_name, metadata=config)
             else:
                 embeddings_collection = embeddings_db.create_collection(coll_name, metadata=config)
-            texts = list(texts)
+
             embeddings = list([None] * len(texts))
             uncached_texts = []
             n = 0

linkml_store/index/indexer.py

@@ -36,6 +36,54 @@ def cosine_similarity(vector1, vector2) -> float:
 class Indexer(BaseModel):
     """
     An indexer operates on a collection in order to search for objects.
+
+    You should use a subcllass of this; this can be looked up dynqamically:
+
+    >>> from linkml_store.index import get_indexer
+    >>> indexer = get_indexer("simple")
+
+    You can customize how objects are indexed by passing in a text template.
+    For example, if your collection has objects with "name" and "profession" attributes,
+    you can index them as "{name} {profession}".
+
+    >>> indexer = get_indexer("simple", text_template="{name} :: {profession}")
+
+    By default, python fstrings are assumed.
+
+    We can test this works using the :ref:`object_to_text` method (normally
+    you would never need to call this directly, but it's useful for testing):
+
+    >>> obj = {"name": "John", "profession": "doctor"}
+    >>> indexer.object_to_text(obj)
+    'John :: doctor'
+
+    You can also use Jinja2 templates; this gives more flexibility and logic,
+    e.g. conditional formatting:
+
+    >>> tmpl = "{{name}}{% if profession %} :: {{profession}}{% endif %}"
+    >>> indexer = get_indexer("simple", text_template=tmpl, text_template_syntax=TemplateSyntaxEnum.jinja2)
+    >>> indexer.object_to_text(obj)
+    'John :: doctor'
+    >>> indexer.object_to_text({"name": "John"})
+    'John'
+
+    You can also specify which attributes to index:
+
+    >>> indexer = get_indexer("simple", index_attributes=["name"])
+    >>> indexer.object_to_text(obj)
+    'John'
+
+    The purpose of an indexer is to translate a collection of objects into a collection of objects
+    such as vectors for purposes such as search. Unless you are implementing your own indexer, you
+    generally don't need to use the methods that return vectors, but we can examine their behavior
+    to get a sense of how they work.
+
+    >>> vectors = indexer.objects_to_vectors([{"name": "Aardvark"}, {"name": "Aardwolf"}, {"name": "Zesty"}])
+    >>> assert cosine_similarity(vectors[0], vectors[1]) > cosine_similarity(vectors[0], vectors[2])
+
+    Note you should consult the documentation for the specific indexer you are using for more details on
+    how text is converted to vectors.
+
     """
 
     name: Optional[str] = None
@@ -122,7 +170,9 @@ class Indexer(BaseModel):
         self, query: str, vectors: List[Tuple[str, INDEX_ITEM]], limit: Optional[int] = None
     ) -> List[Tuple[float, Any]]:
         """
-        Search the index for a query string
+        Use the indexer to search against a database of vectors.
+
+        Note: this is a low-level method, typically you would use the :ref:`search` method on a :ref:`Collection`.
 
         :param query: The query string to search for
         :param vectors: A list of indexed items, where each item is a tuple of (id, vector)

linkml_store/inference/evaluation.py

@@ -0,0 +1,195 @@
+import logging
+from collections.abc import Callable
+from typing import Any, List, Optional
+
+import numpy as np
+import pandas as pd
+from pydantic import BaseModel
+
+from linkml_store.inference import InferenceEngine
+from linkml_store.utils.object_utils import select_nested
+
+logger = logging.getLogger(__name__)
+
+
+def score_match(target: Optional[Any], candidate: Optional[Any], match_function: Optional[Callable] = None) -> float:
+    """
+    Compute a score for a match between two objects
+
+    >>> score_match("a", "a")
+    1.0
+    >>> score_match("a", "b")
+    0.0
+    >>> score_match("abcd", "abcde")
+    0.0
+    >>> score_match("a", None)
+    0.0
+    >>> score_match(None, "a")
+    0.0
+    >>> score_match(None, None)
+    1.0
+    >>> score_match(["a", "b"], ["a", "b"])
+    1.0
+    >>> score_match(["a", "b"], ["b", "a"])
+    1.0
+    >>> round(score_match(["a"], ["b", "a"]), 2)
+    0.67
+    >>> score_match({"a": 1}, {"a": 1})
+    1.0
+    >>> score_match({"a": 1}, {"a": 2})
+    0.0
+    >>> score_match({"a": 1, "b": None}, {"a": 1})
+    1.0
+    >>> score_match([{"a": 1, "b": 2}, {"a": 3, "b": 4}], [{"a": 1, "b": 2}, {"a": 3, "b": 4}])
+    1.0
+    >>> score_match([{"a": 1, "b": 4}, {"a": 3, "b": 2}], [{"a": 1, "b": 2}, {"a": 3, "b": 4}])
+    0.5
+    >>> def char_match(x, y):
+    ...    return len(set(x).intersection(set(y))) / len(set(x).union(set(y)))
+    >>> score_match("abcd", "abc", char_match)
+    0.75
+    >>> score_match(["abcd", "efgh"], ["ac", "gh"], char_match)
+    0.5
+
+
+    :param target:
+    :param candidate:
+    :param match_function: defaults to struct
+    :return:
+    """
+    if target == candidate:
+        return 1.0
+    if target is None or candidate is None:
+        return 0.0
+    if isinstance(target, (set, list)) and isinstance(candidate, (set, list)):
+        # create an all by all matrix using numpy
+        # for each pair of elements, compute the score
+        # return the average score
+        score_matrix = np.array([[score_match(t, c, match_function) for c in candidate] for t in target])
+        best_matches0 = np.max(score_matrix, axis=0)
+        best_matches1 = np.max(score_matrix, axis=1)
+        return (np.sum(best_matches0) + np.sum(best_matches1)) / (len(target) + len(candidate))
+    if isinstance(target, dict) and isinstance(candidate, dict):
+        keys = set(target.keys()).union(candidate.keys())
+        scores = [score_match(target.get(k), candidate.get(k), match_function) for k in keys]
+        return np.mean(scores)
+    if match_function:
+        return match_function(target, candidate)
+    return 0.0
+
+
+class Outcome(BaseModel):
+    true_positive_count: float
+    total_count: int
+
+    @property
+    def accuracy(self) -> float:
+        return self.true_positive_count / self.total_count
+
+
+def evaluate_predictor(
+    predictor: InferenceEngine,
+    target_attributes: List[str],
+    feature_attributes: Optional[List[str]] = None,
+    test_data: pd.DataFrame = None,
+    evaluation_count: Optional[int] = 10,
+    match_function: Optional[Callable] = None,
+) -> Outcome:
+    """
+    Evaluate a predictor by comparing its predictions to the expected values in the testing data.
+
+    :param predictor:
+    :param target_attributes:
+    :param feature_attributes:
+    :param evaluation_count: max iterations
+    :param match_function: function to use for matching
+    :return:
+    """
+    n = 0
+    tp = 0
+    if test_data is None:
+        test_data = predictor.testing_data.as_dataframe()
+    for row in test_data.to_dict(orient="records"):
+        expected_obj = select_nested(row, target_attributes)
+        if feature_attributes:
+            test_obj = {k: v for k, v in row.items() if k not in target_attributes}
+        else:
+            test_obj = row
+        result = predictor.derive(test_obj)
+        tp += score_match(result.predicted_object, expected_obj, match_function)
+        logger.info(f"TP={tp} MF={match_function} Predicted: {result.predicted_object} Expected: {expected_obj}")
+        n += 1
+        if evaluation_count is not None and n >= evaluation_count:
+            break
+    return Outcome(true_positive_count=tp, total_count=n)
+
+
+def score_text_overlap(str1: Any, str2: Any) -> float:
+    """
+    Compute the overlap score between two strings.
+
+    >>> score_text_overlap("abc", "bcde")
+    0.5
+
+    :param str1:
+    :param str2:
+    :return:
+    """
+    if str1 == str2:
+        return 1.0
+    if not str1 or not str2:
+        return 0.0
+    overlap, length = find_longest_overlap(str1, str2)
+    return len(overlap) / max(len(str1), len(str2))
+
+
+def find_longest_overlap(str1: str, str2: str):
+    """
+    Find the longest overlapping substring between two strings.
+
+    Args:
+    str1 (str): The first string
+    str2 (str): The second string
+
+    Returns:
+    tuple: A tuple containing the longest overlapping substring and its length
+
+    Examples:
+    >>> find_longest_overlap("hello world", "world of programming")
+    ('world', 5)
+    >>> find_longest_overlap("abcdefg", "defghi")
+    ('defg', 4)
+    >>> find_longest_overlap("python", "java")
+    ('', 0)
+    >>> find_longest_overlap("", "test")
+    ('', 0)
+    >>> find_longest_overlap("aabbcc", "ddeeff")
+    ('', 0)
+    >>> find_longest_overlap("programming", "PROGRAMMING")
+    ('', 0)
+    """
+    if not str1 or not str2:
+        return "", 0
+
+    # Create a table to store lengths of matching substrings
+    m, n = len(str1), len(str2)
+    dp = [[0] * (n + 1) for _ in range(m + 1)]
+
+    # Variables to store the maximum length and ending position
+    max_length = 0
+    end_pos = 0
+
+    # Fill the dp table
+    for i in range(1, m + 1):
+        for j in range(1, n + 1):
+            if str1[i - 1] == str2[j - 1]:
+                dp[i][j] = dp[i - 1][j - 1] + 1
+                if dp[i][j] > max_length:
+                    max_length = dp[i][j]
+                    end_pos = i
+
+    # Extract the longest common substring
+    start_pos = end_pos - max_length
+    longest_substring = str1[start_pos:end_pos]
+
+    return longest_substring, max_length