linkml-store 0.1.10__py3-none-any.whl → 0.1.12__py3-none-any.whl

linkml_store/cli.py

@@ -16,6 +16,7 @@ from linkml_store.index.implementations.simple_indexer import SimpleIndexer
 from linkml_store.index.indexer import Indexer
 from linkml_store.utils.format_utils import Format, guess_format, load_objects, render_output, write_output
 from linkml_store.utils.object_utils import object_path_update
+from linkml_store.utils.pandas_utils import facet_summary_to_dataframe_unmelted
 
 index_type_option = click.option(
     "--index-type",
@@ -87,6 +88,7 @@ include_internal_option = click.option("--include-internal/--no-include-internal
 @click.option("--set", help="Metadata settings in the form PATHEXPR=value", multiple=True)
 @click.option("-v", "--verbose", count=True)
 @click.option("-q", "--quiet/--no-quiet")
+@click.option("--base-dir", "-B", help="Base directory for the client configuration")
 @click.option(
     "--stacktrace/--no-stacktrace",
     default=False,
@@ -94,7 +96,7 @@ include_internal_option = click.option("--include-internal/--no-include-internal
     help="If set then show full stacktrace on error",
 )
 @click.pass_context
-def cli(ctx, verbose: int, quiet: bool, stacktrace: bool, database, collection, config, set):
+def cli(ctx, verbose: int, quiet: bool, stacktrace: bool, database, collection, config, set, **kwargs):
     """A CLI for interacting with the linkml-store."""
     if not stacktrace:
         sys.tracebacklimit = 0
@@ -117,7 +119,7 @@ def cli(ctx, verbose: int, quiet: bool, stacktrace: bool, database, collection,
     if quiet:
         logger.setLevel(logging.ERROR)
     ctx.ensure_object(dict)
-    client = Client().from_config(config) if config else Client()
+    client = Client().from_config(config, **kwargs) if config else Client()
     settings = ContextSettings(client=client, database_name=database, collection_name=collection)
     ctx.obj["settings"] = settings
     # DEPRECATED
@@ -150,7 +152,7 @@ def cli(ctx, verbose: int, quiet: bool, stacktrace: bool, database, collection,
         #    raise ValueError("Collection must be specified if there are multiple collections.")
         if settings.database and settings.database.list_collections():
             collection = settings.database.list_collections()[0]
-            settings.collection_name = collection.name
+            settings.collection_name = collection.alias
 
 
 @cli.command()
@@ -180,15 +182,15 @@ def insert(ctx, files, object, format):
             objects = load_objects(file_path, format=format)
         else:
             objects = load_objects(file_path)
-        logger.info(f"Inserting {len(objects)} objects from {file_path} into collection '{collection.name}'.")
+        logger.info(f"Inserting {len(objects)} objects from {file_path} into collection '{collection.alias}'.")
         collection.insert(objects)
-        click.echo(f"Inserted {len(objects)} objects from {file_path} into collection '{collection.name}'.")
+        click.echo(f"Inserted {len(objects)} objects from {file_path} into collection '{collection.alias}'.")
     if object:
         for object_str in object:
             logger.info(f"Parsing: {object_str}")
             objects = yaml.safe_load(object_str)
             collection.insert(objects)
-            click.echo(f"Inserted {len(objects)} objects from {object_str} into collection '{collection.name}'.")
+            click.echo(f"Inserted {len(objects)} objects from {object_str} into collection '{collection.alias}'.")
     collection.commit()
 
 
@@ -226,7 +228,11 @@ def store(ctx, files, object, format):
 @click.pass_context
 @click.argument("files", type=click.Path(exists=True), nargs=-1)
 def import_database(ctx, files, format):
-    """Imports a database from a dump."""
+    """Imports a database from a dump.
+
+    See the `export` command for a full list of supported formats. The same
+    formats are generally supported for imports.
+    """
     settings = ctx.obj["settings"]
     db = settings.database
     if not files and not object:
@@ -240,7 +246,30 @@ def import_database(ctx, files, format):
 @click.option("--output", "-o", required=True, type=click.Path(), help="Output file path")
 @click.pass_context
 def export(ctx, output_type, output):
-    """Exports a database to a dump."""
+    """Exports a database to a standard dump format.
+
+    Example:
+
+        linkml-store -d duckdb:///countries.db export -O yaml -o countries.yaml
+
+    Export format will be guessed from extension if not specified
+
+    Example:
+
+        linkml-store -d duckdb:///countries.db export -o countries.json
+
+    Tree formats such as YAML and JSON can natively store an entire database; each collection
+    will be a distinct key in the database.
+
+    Additionally, native dump formats can be used:
+
+    Example:
+
+        linkml-store -d duckdb:///countries.db export -o countries -O duckdb
+
+    Here, `countries` is a directory. This is equivalent to running EXPORT DATABASE
+    (see https://duckdb.org/docs/sql/statements/export.html)
+    """
     settings = ctx.obj["settings"]
     db = settings.database
     if output_type is None:
@@ -324,7 +353,7 @@ def query(ctx, where, limit, output_type, output):
     """
     collection = ctx.obj["settings"].collection
     where_clause = yaml.safe_load(where) if where else None
-    query = Query(from_table=collection.name, where_clause=where_clause, limit=limit)
+    query = Query(from_table=collection.alias, where_clause=where_clause, limit=limit)
     result = collection.query(query)
     output_data = render_output(result.rows, output_type)
     if output:
@@ -341,7 +370,7 @@ def query(ctx, where, limit, output_type, output):
 def list_collections(ctx, **kwargs):
     db = ctx.obj["settings"].database
     for collection in db.list_collections(**kwargs):
-        click.echo(collection.name)
+        click.echo(collection.alias)
         click.echo(render_output(collection.metadata))
 
 
@@ -351,8 +380,9 @@ def list_collections(ctx, **kwargs):
 @click.option("--output-type", "-O", type=format_choice, default="json", help="Output format")
 @click.option("--output", "-o", type=click.Path(), help="Output file path")
 @click.option("--columns", "-S", help="Columns to facet on")
+@click.option("--wide/--no-wide", "-U/--no-U", default=False, show_default=True, help="Wide table")
 @click.pass_context
-def fq(ctx, where, limit, columns, output_type, output):
+def fq(ctx, where, limit, columns, output_type, wide, output):
     """
     Query facets from the specified collection.
 
@@ -379,11 +409,22 @@ def fq(ctx, where, limit, columns, output_type, output):
             return "+".join([str(x) for x in key])
         return key
 
-    count_dict = {}
-    for key, value in results.items():
-        value_as_dict = {_untuple(v[0:-1]): v[-1] for v in value}
-        count_dict[_untuple(key)] = value_as_dict
-    output_data = render_output(count_dict, output_type)
+    if wide:
+        results_obj = facet_summary_to_dataframe_unmelted(results)
+    else:
+        if output_type == Format.PYTHON.value:
+            results_obj = results
+        elif output_type in [Format.TSV.value, Format.CSV.value]:
+            results_obj = []
+            for fc, data in results.items():
+                for v, c in data:
+                    results_obj.append({"facet": fc, "value": v, "count": c})
+        else:
+            results_obj = {}
+            for key, value in results.items():
+                value_as_dict = {_untuple(v[0:-1]): v[-1] for v in value}
+                results_obj[_untuple(key)] = value_as_dict
+    output_data = render_output(results_obj, output_type)
     if output:
         with open(output, "w") as f:
             f.write(output_data)
@@ -403,14 +444,17 @@ def _get_index(index_type=None, **kwargs) -> Indexer:
 @click.option("--where", "-w", type=click.STRING, help="WHERE clause for the query")
 @click.option("--output-type", "-O", type=format_choice, default=Format.FORMATTED.value, help="Output format")
 @click.option("--output", "-o", type=click.Path(), help="Output file path")
+@click.option(
+    "--limit", "-l", default=-1, show_default=True, type=click.INT, help="Maximum number of results to return"
+)
 @click.pass_context
-def describe(ctx, where, output_type, output):
+def describe(ctx, where, output_type, output, limit):
     """
     Describe the collection schema.
     """
     where_clause = yaml.safe_load(where) if where else None
     collection = ctx.obj["settings"].collection
-    df = collection.find(where_clause, limit=1).rows_dataframe
+    df = collection.find(where_clause, limit=limit).rows_dataframe
     write_output(df.describe(include="all").transpose(), output_type, target=output)
 
 
@@ -468,7 +512,7 @@ def search(ctx, search_term, where, limit, index_type, output_type, output, auto
     """Search objects in the specified collection."""
     collection = ctx.obj["settings"].collection
     ix = get_indexer(index_type)
-    logger.info(f"Attaching index to collection {collection.name}: {ix.model_dump()}")
+    logger.info(f"Attaching index to collection {collection.alias}: {ix.model_dump()}")
     collection.attach_indexer(ix, auto_index=auto_index)
     result = collection.search(search_term, where=where, limit=limit)
     output_data = render_output([{"score": row[0], **row[1]} for row in result.ranked_rows], output_type)
@@ -498,6 +542,7 @@ def indexes(ctx):
 def validate(ctx, output_type, output):
     """Validate objects in the specified collection."""
     collection = ctx.obj["settings"].collection
+    logger.info(f"Validating collection {collection.alias}")
     validation_results = [json_dumper.to_dict(x) for x in collection.iter_validate_collection()]
     output_data = render_output(validation_results, output_type)
     if output:

linkml_store/index/__init__.py

@@ -1,3 +1,14 @@
+"""
+Indexers package.
+
+Indexers allow indexes to be added to existing :class:`Collection` objects.
+
+Current two are supported:
+
+* simple: :class:`SimpleIndexer`
+* llm: :class:`LLMIndexer`
+"""
+
 from typing import Type
 
 from linkml_store.index.implementations.llm_indexer import LLMIndexer
@@ -14,7 +25,7 @@ def get_indexer_class(name: str) -> Type[Indexer]:
     """
     Get an indexer class by name.
 
-    :param name: the name of the indexer
+    :param name: the name of the indexer (simple, llm, ...)
     :return: the indexer class
     """
     if name not in INDEXER_CLASSES:
@@ -26,7 +37,10 @@ def get_indexer(index_type: str, **kwargs) -> Indexer:
     """
     Get an indexer by name.
 
-    :param name: the name of the indexer
+    >>> simple_indexer = get_indexer("simple")
+    >>> llm_indexer = get_indexer("llm")
+
+    :param name: the name of the indexer (simple, llm, ...)
     :param kwargs: additional arguments to pass to the indexer
     :return: the indexer
     """

linkml_store/index/implementations/llm_indexer.py

@@ -74,7 +74,7 @@ class LLMIndexer(Indexer):
 
             embeddings_client = Client()
             config = CollectionConfig(
-                name=coll_name,
+                alias=coll_name,
                 type="Embeddings",
                 attributes={
                     "text": {"range": "string"},
@@ -116,6 +116,7 @@ class LLMIndexer(Indexer):
                     embeddings_collection.insert(
                         {"text": uncached_texts[i], "embedding": embeddings[index], "model_id": model_id}
                     )
+                embeddings_collection.commit()
         else:
             logger.info(f"Embedding {len(texts)} texts")
             embeddings = model.embed_multi(texts)

linkml_store/index/indexer.py

@@ -11,11 +11,22 @@ logger = logging.getLogger(__name__)
 
 
 class TemplateSyntaxEnum(str, Enum):
+    """
+    Template syntax types.
+    """
+
     jinja2 = "jinja2"
     fstring = "fstring"
 
 
-def cosine_similarity(vector1, vector2):
+def cosine_similarity(vector1, vector2) -> float:
+    """
+    Calculate the cosine similarity between two vectors
+
+    :param vector1:
+    :param vector2:
+    :return:
+    """
     dot_product = np.dot(vector1, vector2)
     norm1 = np.linalg.norm(vector1)
     norm2 = np.linalg.norm(vector2)
@@ -24,7 +35,7 @@ def cosine_similarity(vector1, vector2):
 
 class Indexer(BaseModel):
     """
-    An index operates on a collection in order to search for objects.
+    An indexer operates on a collection in order to search for objects.
     """
 
     name: Optional[str] = None

linkml_store/utils/file_utils.py

@@ -0,0 +1,37 @@
+import logging
+import shutil
+import tempfile
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+# Set up logging
+logger = logging.getLogger(__name__)
+
+
+def safe_remove_directory(dir_path: Path, no_backup: bool = False) -> Optional[Path]:
+    # Ensure the directory exists
+    if not dir_path.exists():
+        raise FileNotFoundError(f"Directory does not exist: {dir_path}")
+    try:
+        if no_backup:
+            # Move to a temporary directory instead of permanent removal
+            with tempfile.TemporaryDirectory() as tmpdir:
+                tmp_path = Path(tmpdir) / dir_path.name
+                shutil.move(str(dir_path), str(tmp_path))
+                logger.info(f"Directory moved to temporary location: {tmp_path}")
+                # The directory will be automatically removed when exiting the context manager
+            return None
+        else:
+            # Create a backup directory name with timestamp
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            backup_dir = dir_path.with_name(f"{dir_path.name}_backup_{timestamp}")
+
+            # Move the directory to the backup location
+            shutil.move(str(dir_path), str(backup_dir))
+            logger.info(f"Directory backed up to: {backup_dir}")
+            return backup_dir
+
+    except Exception as e:
+        logger.error(f"An error occurred: {e}")
+        return None

linkml_store/utils/format_utils.py

@@ -1,14 +1,22 @@
 import csv
+import gzip
+import io
 import json
+import logging
 import sys
+import tarfile
 from enum import Enum
 from io import StringIO
 from pathlib import Path
-from typing import Any, Dict, List, Optional, TextIO, Type, Union
+from typing import IO, Any, Dict, List, Optional, TextIO, Type, Union
 
 import pandas as pd
+import pystow
 import yaml
 from pydantic import BaseModel
+from tabulate import tabulate
+
+logger = logging.getLogger(__name__)
 
 
 class Format(Enum):
@@ -21,12 +29,163 @@ class Format(Enum):
     YAML = "yaml"
     TSV = "tsv"
     CSV = "csv"
+    PYTHON = "python"
     PARQUET = "parquet"
     FORMATTED = "formatted"
+    TABLE = "table"
+    SQLDUMP_DUCKDB = "duckdb"
+    SQLDUMP_POSTGRES = "postgres"
+    DUMP_MONGODB = "mongodb"
+
+    @classmethod
+    def guess_format(cls, file_name: str) -> Optional["Format"]:
+        ext = Path(file_name).suffix.lower()
+
+        format_map = {
+            ".json": cls.JSON,
+            ".jsonl": cls.JSONL,
+            ".yaml": cls.YAML,
+            ".yml": cls.YAML,
+            ".tsv": cls.TSV,
+            ".csv": cls.CSV,
+            ".py": cls.PYTHON,
+            ".parquet": cls.PARQUET,
+            ".pq": cls.PARQUET,
+        }
+        fmt = format_map.get(ext, None)
+        if fmt is None:
+            if ext.startswith("."):
+                ext = ext[1:]
+            if ext in [f.value for f in Format]:
+                return Format(ext)
+        return fmt
+
+    def is_dump_format(self):
+        return self in [Format.SQLDUMP_DUCKDB, Format.SQLDUMP_POSTGRES, Format.DUMP_MONGODB]
+
+
+def load_objects_from_url(
+    url: str,
+    format: Union[Format, str] = None,
+    expected_type: Type = None,
+    local_path: Optional[str] = None,
+    **kwargs,
+) -> List[Dict[str, Any]]:
+    """
+    Load objects from a URL in JSON, JSONLines, YAML, CSV, or TSV format.
+
+    :param url: The URL to the file.
+    :param format: The format of the file. Can be a Format enum or a string value.
+    :param expected_type: The target type to load the objects into.
+    :param local_path: The local path to save the file to.
+    :return: A list of dictionaries representing the loaded objects.
+    """
+    local_path = pystow.ensure("linkml", "linkml-store", url=url)
+    logger.info(f"synced to {local_path}")
+    objs = load_objects(local_path, format=format, expected_type=expected_type, **kwargs)
+    if not objs:
+        raise ValueError(f"No objects loaded from URL: {url}")
+    return objs
+
+
+def process_file(
+    f: IO, format: Format, expected_type: Optional[Type] = None, header_comment_token: Optional[str] = None
+) -> List[Dict[str, Any]]:
+    """
+    Process a single file and return a list of objects.
+    """
+    if format == Format.JSON:
+        objs = json.load(f)
+    elif format == Format.JSONL:
+        objs = [json.loads(line) for line in f]
+    elif format == Format.YAML:
+        if expected_type and expected_type == list:  # noqa E721
+            objs = list(yaml.safe_load_all(f))
+        else:
+            objs = yaml.safe_load(f)
+    elif format in [Format.TSV, Format.CSV]:
+        if header_comment_token:
+            while True:
+                pos = f.tell()
+                line = f.readline()
+                if not line.startswith(header_comment_token):
+                    f.seek(pos)
+                    break
+        delimiter = "\t" if format == Format.TSV else ","
+        reader = csv.DictReader(f, delimiter=delimiter)
+        objs = list(reader)
+    elif format == Format.PARQUET:
+        import pyarrow.parquet as pq
+
+        table = pq.read_table(f)
+        objs = table.to_pandas().to_dict(orient="records")
+    elif format in [Format.PYTHON, Format.FORMATTED, Format.TABLE]:
+        raise ValueError(f"Format {format} is not supported for loading objects")
+    else:
+        raise ValueError(f"Unsupported file format: {format}")
+
+    if not isinstance(objs, list):
+        objs = [objs]
+    return objs
 
 
 def load_objects(
-    file_path: Union[str, Path], format: Union[Format, str] = None, expected_type: Type = None
+    file_path: Union[str, Path],
+    format: Optional[Union[Format, str]] = None,
+    compression: Optional[str] = None,
+    expected_type: Optional[Type] = None,
+    header_comment_token: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Load objects from a file or archive in supported formats.
+    For tgz archives, it processes all files and concatenates the results.
+
+    :param file_path: The path to the file or archive.
+    :param format: The format of the file. Can be a Format enum or a string value.
+    :param compression: The compression type. Supports 'gz' for gzip and 'tgz' for tar.gz.
+    :param expected_type: The target type to load the objects into, e.g. list
+    :param header_comment_token: Token used for header comments to be skipped
+    :return: A list of dictionaries representing the loaded objects.
+    """
+    if isinstance(file_path, Path):
+        file_path = str(file_path)
+
+    if isinstance(format, str):
+        format = Format(format)
+
+    all_objects = []
+
+    if compression == "tgz":
+        with tarfile.open(file_path, "r:gz") as tar:
+            for member in tar.getmembers():
+                if member.isfile():
+                    f = tar.extractfile(member)
+                    if f:
+                        content = io.TextIOWrapper(f)
+                        member_format = Format.guess_format(member.name) if not format else format
+                        logger.debug(f"Processing tar member {member.name} with format {member_format}")
+                        all_objects.extend(process_file(content, member_format, expected_type, header_comment_token))
+    else:
+        if Path(file_path).is_dir():
+            raise ValueError(f"{file_path} is a dir, which is invalid for {format}")
+        mode = "rb" if format == Format.PARQUET or compression == "gz" else "r"
+        open_func = gzip.open if compression == "gz" else open
+        format = Format.guess_format(file_path) if not format else format
+        with open_func(file_path, mode) if file_path != "-" else sys.stdin as f:
+            if compression == "gz" and mode == "r":
+                f = io.TextIOWrapper(f)
+            all_objects = process_file(f, format, expected_type, header_comment_token)
+
+    logger.debug(f"Loaded {len(all_objects)} objects from {file_path}")
+    return all_objects
+
+
+def xxxload_objects(
+    file_path: Union[str, Path],
+    format: Union[Format, str] = None,
+    compression: Optional[str] = None,
+    expected_type: Type = None,
+    header_comment_token: Optional[str] = None,
 ) -> List[Dict[str, Any]]:
     """
     Load objects from a file in JSON, JSONLines, YAML, CSV, or TSV format.
@@ -37,7 +196,7 @@ def load_objects(
 
     :param file_path: The path to the file.
     :param format: The format of the file. Can be a Format enum or a string value.
-    :param expected_type: The target type to load the objects into.
+    :param expected_type: The target type to load the objects into, e.g. list
     :return: A list of dictionaries representing the loaded objects.
     """
     if isinstance(format, str):
@@ -48,6 +207,12 @@ def load_objects(
 
     if not format and (file_path.endswith(".parquet") or file_path.endswith(".pq")):
         format = Format.PARQUET
+    if not format and file_path.endswith(".tsv"):
+        format = Format.TSV
+    if not format and file_path.endswith(".csv"):
+        format = Format.CSV
+    if not format and file_path.endswith(".py"):
+        format = Format.PYTHON
 
     mode = "r"
     if format == Format.PARQUET:
@@ -68,11 +233,29 @@ def load_objects(
             objs = list(yaml.safe_load_all(f))
         else:
             objs = yaml.safe_load(f)
-    elif format == Format.TSV or (not format and file_path.endswith(".tsv")):
-        reader = csv.DictReader(f, delimiter="\t")
-        objs = list(reader)
-    elif format == Format.CSV or (not format and file_path.endswith(".csv")):
-        reader = csv.DictReader(f)
+    elif format == Format.TSV or format == Format.CSV:
+        # Skip initial comment lines if comment_char is set
+        if header_comment_token:
+            # Store the original position
+            original_pos = f.tell()
+
+            # Read and store lines until we find a non-comment line
+            lines = []
+            for line in f:
+                if not line.startswith(header_comment_token):
+                    break
+                lines.append(line)
+
+            # Go back to the original position
+            f.seek(original_pos)
+
+            # Skip the comment lines we found
+            for _ in lines:
+                f.readline()
+        if format == Format.TSV:
+            reader = csv.DictReader(f, delimiter="\t")
+        else:
+            reader = csv.DictReader(f)
         objs = list(reader)
     elif format == Format.PARQUET:
         import pyarrow.parquet as pq
@@ -118,7 +301,7 @@ def write_output(
 
 
 def render_output(
-    data: Union[List[Dict[str, Any]], Dict[str, Any], pd.DataFrame], format: Union[Format, str] = Format.YAML
+    data: Union[List[Dict[str, Any]], Dict[str, Any], pd.DataFrame], format: Optional[Union[Format, str]] = Format.YAML
 ) -> str:
     """
     Render output data in JSON, JSONLines, YAML, CSV, or TSV format.
@@ -151,6 +334,9 @@ def render_output(
     if isinstance(data, pd.DataFrame):
         data = data.to_dict(orient="records")
 
+    if isinstance(data, dict) and format in [Format.TSV, Format.CSV]:
+        data = [data]
+
     if isinstance(data, BaseModel):
         data = data.model_dump()
 
@@ -158,6 +344,10 @@ def render_output(
         return json.dumps(data, indent=2, default=str)
     elif format == Format.JSONL:
         return "\n".join(json.dumps(obj) for obj in data)
+    elif format == Format.PYTHON:
+        return str(data)
+    elif format == Format.TABLE:
+        return tabulate(pd.DataFrame(data), headers="keys", tablefmt="psql")
     elif format == Format.YAML:
         if isinstance(data, list):
             return yaml.safe_dump_all(data, sort_keys=False)
@@ -210,15 +400,4 @@ def guess_format(path: str) -> Optional[Format]:
     :param path: The path to the file.
     :return: The guessed format.
     """
-    if path.endswith(".json"):
-        return Format.JSON
-    elif path.endswith(".jsonl"):
-        return Format.JSONL
-    elif path.endswith(".yaml") or path.endswith(".yml"):
-        return Format.YAML
-    elif path.endswith(".tsv"):
-        return Format.TSV
-    elif path.endswith(".csv"):
-        return Format.CSV
-    else:
-        return None
+    return Format.guess_format(path)