linkml-store 0.2.6__py3-none-any.whl → 0.2.9__py3-none-any.whl

linkml_store/api/client.py

@@ -12,7 +12,6 @@ from linkml_store.api.config import ClientConfig
 logger = logging.getLogger(__name__)
 
 
-
 HANDLE_MAP = {
     "duckdb": "linkml_store.api.stores.duckdb.duckdb_database.DuckDBDatabase",
     "sqlite": "linkml_store.api.stores.duckdb.duckdb_database.DuckDBDatabase",
@@ -220,14 +219,14 @@ class Client:
             scheme, _ = handle.split(":", 1)
         if scheme not in HANDLE_MAP:
             raise ValueError(f"Unknown scheme: {scheme}")
-        module_path, class_name = HANDLE_MAP[scheme].rsplit('.', 1)
+        module_path, class_name = HANDLE_MAP[scheme].rsplit(".", 1)
         try:
             module = importlib.import_module(module_path)
             cls = getattr(module, class_name)
         except ImportError as e:
             raise ImportError(f"Failed to import {scheme} database. Make sure the correct extras are installed: {e}")
 
-        #cls = HANDLE_MAP[scheme]
+        # cls = HANDLE_MAP[scheme]
         db = cls(handle=handle, recreate_if_exists=recreate_if_exists, **kwargs)
         if schema_view:
             db.set_schema_view(schema_view)

linkml_store/api/collection.py

@@ -211,7 +211,7 @@ class Collection(Generic[DatabaseType]):
         """
         raise NotImplementedError
 
-    def index (
+    def index(
         self,
         objs: Union[OBJECT, List[OBJECT]],
         index_name: Optional[str] = None,
@@ -231,10 +231,13 @@ class Collection(Generic[DatabaseType]):
         """
         raise NotImplementedError
 
-    def upsert(self,
-               objs: Union[OBJECT, List[OBJECT]],
-               filter_fields: List[str],
-               update_fields: Union[List[str], None] = None, **kwargs):
+    def upsert(
+        self,
+        objs: Union[OBJECT, List[OBJECT]],
+        filter_fields: List[str],
+        update_fields: Union[List[str], None] = None,
+        **kwargs,
+    ):
         """
         Add one or more objects to the collection.
 
@@ -454,7 +457,12 @@ class Collection(Generic[DatabaseType]):
             return qr.rows[0]
         return None
 
-    def find(self, where: Optional[Any] = None, **kwargs) -> QueryResult:
+    def find(
+        self,
+        where: Optional[Any] = None,
+        select_cols: Optional[List[str]] = None,
+        **kwargs,
+    ) -> QueryResult:
         """
         Find objects in the collection using a where query.
 
@@ -484,10 +492,14 @@ class Collection(Generic[DatabaseType]):
 
 
         :param where:
+        :param select_cols:
         :param kwargs:
         :return:
         """
-        query = self._create_query(where_clause=where)
+        query = self._create_query(
+            where_clause=where,
+            select_cols=select_cols,
+        )
         self._pre_query_hook(query)
         return self.query(query, **kwargs)
 
@@ -587,6 +599,7 @@ class Collection(Generic[DatabaseType]):
                 assert ix_coll.size() > 0
         qr = ix_coll.find(where=where, limit=-1, **kwargs)
         index_col = ix.index_field
+
         # TODO: optimize this for large indexes
         def row2array(row):
             v = row[index_col]
@@ -594,6 +607,7 @@ class Collection(Generic[DatabaseType]):
                 # sqlite stores arrays as strings
                 v = json.loads(v)
             return np.array(v, dtype=float)
+
         vector_pairs = [(row, row2array(row)) for row in qr.rows]
         results = ix.search(query, vector_pairs, limit=limit, mmr_relevance_factor=mmr_relevance_factor, **kwargs)
         for r in results:
@@ -608,6 +622,47 @@ class Collection(Generic[DatabaseType]):
         new_qr.rows = [r[1] for r in results]
         return new_qr
 
+    def group_by(
+        self,
+        group_by_fields: List[str],
+        inlined_field="objects",
+        agg_map: Optional[Dict[str, str]] = None,
+        where: Optional[Dict] = None,
+        **kwargs,
+    ) -> QueryResult:
+        """
+        Group objects in the collection by a column.
+
+        :param group_by:
+        :param where:
+        :param kwargs:
+        :return:
+        """
+        if isinstance(group_by_fields, str):
+            group_by_fields = [group_by_fields]
+        df = self.find(where=where, limit=-1).rows_dataframe
+        
+        # Handle the case where agg_map is None
+        if agg_map is None:
+            agg_map = {}
+            
+        pk_fields = agg_map.get("first", []) + group_by_fields
+        list_fields = agg_map.get("list", [])
+        if not list_fields:
+            list_fields = [a for a in df.columns if a not in pk_fields]
+
+        grouped_objs = defaultdict(list)
+        for _, row in df.iterrows():
+            pk = tuple(row[pk_fields])
+            grouped_objs[pk].append({k: row[k] for k in list_fields})
+        results = []
+        for pk, objs in grouped_objs.items():
+            top_obj = {k: v for k, v in zip(pk_fields, pk)}
+            top_obj[inlined_field] = objs
+            results.append(top_obj)
+        r = QueryResult(num_rows=len(results), rows=results)
+        return r
+
     @property
     def is_internal(self) -> bool:
         """
@@ -1062,7 +1117,7 @@ class Collection(Generic[DatabaseType]):
             multivalued = any(multivalueds)
             inlined = any(inlineds)
             if multivalued and False in multivalueds:
-                raise ValueError(f"Mixed list non list: {vs} // inferred= {multivalueds}")
+                logger.info(f"Mixed list non list: {vs} // inferred= {multivalueds}")
             # if not rngs:
             #    raise AssertionError(f"Empty rngs for {k} = {vs}")
             rng = rngs[0] if rngs else None

linkml_store/api/database.py

@@ -595,7 +595,31 @@ class Database(ABC, Generic[CollectionType]):
             sb.add_class(coll.target_class_name)
         return SchemaView(sb.schema)
 
-    def iter_validate_database(self, **kwargs) -> Iterator["ValidationResult"]:
+    def validate_database(self, **kwargs) -> List["ValidationResult"]:
+        """
+        Validate the contents of the database.
+
+        As `iter_validate_database`, but returns a list of validation results.
+
+        :param kwargs:
+        :return:
+        """
+        return list(self.iter_validate_database(**kwargs))
+
+    def validate_database(self, **kwargs) -> List["ValidationResult"]:
+        """
+        Validate the contents of the database.
+
+        As `iter_validate_database`, but returns a list of validation results.
+
+        :param kwargs:
+        :return:
+        """
+        return list(self.iter_validate_database(**kwargs))
+
+    def iter_validate_database(
+        self, ensure_referential_integrity: bool = None, **kwargs
+    ) -> Iterator["ValidationResult"]:
         """
         Validate the contents of the database.
 
@@ -635,12 +659,14 @@ class Database(ABC, Generic[CollectionType]):
         'capital' is a required property
         'continent' is a required proper
 
+        :param ensure_referential_integrity: ensure referential integrity
         :param kwargs:
         :return: iterator over validation results
         """
         for collection in self.list_collections():
             yield from collection.iter_validate_collection(**kwargs)
-        if self.metadata.ensure_referential_integrity:
+        if self.metadata.ensure_referential_integrity or ensure_referential_integrity:
+            logger.info(f"Validating referential integrity on {self.alias}")
             yield from self._validate_referential_integrity(**kwargs)
 
     def _validate_referential_integrity(self, **kwargs) -> Iterator["ValidationResult"]:
@@ -661,7 +687,9 @@ class Database(ABC, Generic[CollectionType]):
             induced_slots = sv.class_induced_slots(cd.name)
             slot_map = {s.name: s for s in induced_slots}
             # rmap = {s.name: s.range for s in induced_slots}
+            # map slot ranges to a collection where that range is stored
             sr_to_coll = {s.name: cmap.get(s.range, []) for s in induced_slots if s.range}
+            logger.debug(f"Validating referential integrity for {collection.target_class_name} // {sr_to_coll}")
             for obj in collection.find_iter():
                 for k, v in obj.items():
                     if k not in sr_to_coll:

linkml_store/api/stores/duckdb/duckdb_collection.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, Tuple
 
 import sqlalchemy as sqla
 from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition
@@ -8,7 +8,7 @@ from sqlalchemy.sql.ddl import CreateTable
 
 from linkml_store.api import Collection
 from linkml_store.api.collection import DEFAULT_FACET_LIMIT, OBJECT
-from linkml_store.api.queries import Query
+from linkml_store.api.queries import Query, QueryResult
 from linkml_store.api.stores.duckdb.mappings import TMAP
 from linkml_store.utils.sql_utils import facet_count_sql
 
@@ -94,7 +94,9 @@ class DuckDBCollection(Collection):
 
     def query_facets(
         self, where: Dict = None, facet_columns: List[str] = None, facet_limit=DEFAULT_FACET_LIMIT, **kwargs
-    ) -> Dict[str, Dict[str, int]]:
+    ) -> Dict[Union[str, Tuple[str, ...]], List[Tuple[Any, int]]]:
+        if facet_limit is None:
+            facet_limit = DEFAULT_FACET_LIMIT
         results = {}
         cd = self.class_definition()
         with self.parent.engine.connect() as conn:
@@ -143,6 +145,166 @@ class DuckDBCollection(Collection):
             return True
         return False
 
+    def group_by(
+        self,
+        group_by_fields: List[str],
+        inlined_field="objects",
+        agg_map: Optional[Dict[str, str]] = None,
+        where: Optional[Dict] = None,
+        **kwargs,
+    ) -> QueryResult:
+        """
+        Group objects in the collection by specified fields using SQLAlchemy.
+        
+        This implementation leverages DuckDB's SQL capabilities for more efficient grouping.
+        
+        :param group_by_fields: List of fields to group by
+        :param inlined_field: Field name to store aggregated objects
+        :param agg_map: Dictionary mapping aggregation types to fields
+        :param where: Filter conditions
+        :param kwargs: Additional arguments
+        :return: Query result containing grouped data
+        """
+        if isinstance(group_by_fields, str):
+            group_by_fields = [group_by_fields]
+            
+        cd = self.class_definition()
+        if not cd:
+            logger.debug(f"No class definition defined for {self.alias} {self.target_class_name}")
+            return super().group_by(group_by_fields, inlined_field, agg_map, where, **kwargs)
+            
+        # Check if the table exists
+        if not self.parent._table_exists(self.alias):
+            logger.debug(f"Table {self.alias} doesn't exist, falling back to parent implementation")
+            return super().group_by(group_by_fields, inlined_field, agg_map, where, **kwargs)
+            
+        # Get table definition
+        table = self._sqla_table(cd)
+        engine = self.parent.engine
+        
+        # Create a SQLAlchemy select statement for groups
+        from sqlalchemy import select, func, and_, or_
+        group_cols = [table.c[field] for field in group_by_fields if field in table.columns.keys()]
+        
+        if not group_cols:
+            logger.warning(f"None of the group_by fields {group_by_fields} found in table columns")
+            return super().group_by(group_by_fields, inlined_field, agg_map, where, **kwargs)
+        
+        stmt = select(*group_cols).distinct()
+        
+        # Add where conditions if specified
+        if where:
+            conditions = []
+            for k, v in where.items():
+                if k in table.columns.keys():
+                    # Handle different operator types (dict values for operators)
+                    if isinstance(v, dict):
+                        for op, val in v.items():
+                            if op == "$gt":
+                                conditions.append(table.c[k] > val)
+                            elif op == "$gte":
+                                conditions.append(table.c[k] >= val)
+                            elif op == "$lt":
+                                conditions.append(table.c[k] < val)
+                            elif op == "$lte":
+                                conditions.append(table.c[k] <= val)
+                            elif op == "$ne":
+                                conditions.append(table.c[k] != val)
+                            elif op == "$in":
+                                conditions.append(table.c[k].in_(val))
+                            else:
+                                # Default to equality for unknown operators
+                                logger.warning(f"Unknown operator {op}, using equality")
+                                conditions.append(table.c[k] == val)
+                    else:
+                        # Direct equality comparison
+                        conditions.append(table.c[k] == v)
+            
+            if conditions:
+                for condition in conditions:
+                    stmt = stmt.where(condition)
+        
+        results = []
+        try:
+            with engine.connect() as conn:
+                # Get all distinct groups
+                group_result = conn.execute(stmt)
+                group_rows = list(group_result)
+                
+                # For each group, get all objects
+                for group_row in group_rows:
+                    # Build conditions for this group
+                    group_conditions = []
+                    group_dict = {}
+                    
+                    for i, field in enumerate(group_by_fields):
+                        if field in table.columns.keys():
+                            value = group_row[i]
+                            group_dict[field] = value
+                            if value is None:
+                                group_conditions.append(table.c[field].is_(None))
+                            else:
+                                group_conditions.append(table.c[field] == value)
+                    
+                    # Get all rows for this group
+                    row_stmt = select(*table.columns)
+                    for condition in group_conditions:
+                        row_stmt = row_stmt.where(condition)
+                    
+                    # Add original where conditions
+                    if where:
+                        for k, v in where.items():
+                            if k in table.columns.keys():
+                                # Handle different operator types for the row query as well
+                                if isinstance(v, dict):
+                                    for op, val in v.items():
+                                        if op == "$gt":
+                                            row_stmt = row_stmt.where(table.c[k] > val)
+                                        elif op == "$gte":
+                                            row_stmt = row_stmt.where(table.c[k] >= val)
+                                        elif op == "$lt":
+                                            row_stmt = row_stmt.where(table.c[k] < val)
+                                        elif op == "$lte":
+                                            row_stmt = row_stmt.where(table.c[k] <= val)
+                                        elif op == "$ne":
+                                            row_stmt = row_stmt.where(table.c[k] != val)
+                                        elif op == "$in":
+                                            row_stmt = row_stmt.where(table.c[k].in_(val))
+                                        else:
+                                            # Default to equality for unknown operators
+                                            row_stmt = row_stmt.where(table.c[k] == val)
+                                else:
+                                    # Direct equality comparison
+                                    row_stmt = row_stmt.where(table.c[k] == v)
+                    
+                    row_result = conn.execute(row_stmt)
+                    rows = list(row_result)
+                    
+                    # Convert rows to dictionaries
+                    objects = []
+                    for row in rows:
+                        obj = {}
+                        for i, col in enumerate(row._fields):
+                            obj[col] = row[i]
+                        objects.append(obj)
+                    
+                    # Apply agg_map to filter fields if specified
+                    if agg_map and "list" in agg_map:
+                        list_fields = agg_map["list"]
+                        if list_fields:
+                            objects = [{k: obj.get(k) for k in list_fields if k in obj} for obj in objects]
+                    
+                    # Create the result object
+                    result_obj = group_dict.copy()
+                    result_obj[inlined_field] = objects
+                    results.append(result_obj)
+                
+                return QueryResult(num_rows=len(results), rows=results)
+        except Exception as e:
+            logger.warning(f"Error in DuckDB group_by: {e}")
+            # Fall back to parent implementation
+            return super().group_by(group_by_fields, inlined_field, agg_map, where, **kwargs)
+
     def _create_table(self, cd: ClassDefinition):
         if self._table_created or self.metadata.is_prepopulated:
             logger.info(f"Already have table for: {cd.name}")

linkml_store/api/stores/duckdb/duckdb_database.py

@@ -100,9 +100,9 @@ class DuckDBDatabase(Database):
             meta_query = Query(
                 from_table="sqlite_master",
                 where_clause={
-                    #"type": "table",
+                    # "type": "table",
                     "name": table,
-                }
+                },
             )
         else:
             if table.startswith("information_schema"):
@@ -112,7 +112,7 @@ class DuckDBDatabase(Database):
                 where_clause={
                     "table_type": "BASE TABLE",
                     "table_name": table,
-                }
+                },
             )
 
         qr = self.query(meta_query)

linkml_store/api/stores/filesystem/__init__.py

@@ -4,7 +4,7 @@ Adapter for FileSystem wrapper
 Handles have the form:
 
  - ``file:<path>`` for a local file
- """
+"""
 
 from linkml_store.api.stores.filesystem.filesystem_collection import FileSystemCollection
 from linkml_store.api.stores.filesystem.filesystem_database import FileSystemDatabase

linkml_store/api/stores/mongodb/mongodb_collection.py

@@ -41,13 +41,14 @@ class MongoDBCollection(Collection):
             del obj["_id"]
         self._post_insert_hook(objs)
 
-
-    def index(self,
-              objs: Union[OBJECT, List[OBJECT]],
-              index_name: Optional[str] = None,
-              replace: bool = False,
-              unique: bool = False,
-              **kwargs):
+    def index(
+        self,
+        objs: Union[OBJECT, List[OBJECT]],
+        index_name: Optional[str] = None,
+        replace: bool = False,
+        unique: bool = False,
+        **kwargs,
+    ):
         """
         Create indexes on the collection.
 
@@ -86,11 +87,13 @@ class MongoDBCollection(Collection):
             else:
                 logging.debug(f"Index already exists for field {obj}, skipping creation.")
 
-    def upsert(self,
-               objs: Union[OBJECT, List[OBJECT]],
-               filter_fields: List[str],
-               update_fields: Optional[List[str]] = None,
-               **kwargs):
+    def upsert(
+        self,
+        objs: Union[OBJECT, List[OBJECT]],
+        filter_fields: List[str],
+        update_fields: Optional[List[str]] = None,
+        **kwargs,
+    ):
         """
         Upsert one or more documents into the MongoDB collection.
 
@@ -164,6 +167,8 @@ class MongoDBCollection(Collection):
         facet_limit=DEFAULT_FACET_LIMIT,
         **kwargs,
     ) -> Dict[Union[str, Tuple[str, ...]], List[Tuple[Any, int]]]:
+        if facet_limit is None:
+            facet_limit = DEFAULT_FACET_LIMIT
         results = {}
         if not facet_columns:
             facet_columns = list(self.class_definition().attributes.keys())
@@ -260,3 +265,101 @@ class MongoDBCollection(Collection):
         if deleted_rows_count == 0 and not missing_ok:
             raise ValueError(f"No rows found for {where}")
         return deleted_rows_count
+        
+    def group_by(
+        self,
+        group_by_fields: List[str],
+        inlined_field="objects",
+        agg_map: Optional[Dict[str, str]] = None,
+        where: Optional[Dict] = None,
+        **kwargs,
+    ) -> QueryResult:
+        """
+        Group objects in the collection by specified fields using MongoDB's aggregation pipeline.
+        
+        This implementation leverages MongoDB's native aggregation capabilities for efficient grouping.
+        
+        :param group_by_fields: List of fields to group by
+        :param inlined_field: Field name to store aggregated objects
+        :param agg_map: Dictionary mapping aggregation types to fields
+        :param where: Filter conditions
+        :param kwargs: Additional arguments
+        :return: Query result containing grouped data
+        """
+        if isinstance(group_by_fields, str):
+            group_by_fields = [group_by_fields]
+        
+        # Build the group key for MongoDB
+        if len(group_by_fields) == 1:
+            # Single field grouping
+            group_id = f"${group_by_fields[0]}"
+        else:
+            # Multi-field grouping
+            group_id = {field: f"${field}" for field in group_by_fields}
+        
+        # Start building the pipeline
+        pipeline = []
+        
+        # Add match stage if where clause is provided
+        if where:
+            pipeline.append({"$match": where})
+        
+        # Add the group stage
+        group_stage = {
+            "$group": {
+                "_id": group_id,
+                "objects": {"$push": "$$ROOT"}
+            }
+        }
+        pipeline.append(group_stage)
+        
+        # Execute the aggregation
+        logger.debug(f"MongoDB group_by pipeline: {pipeline}")
+        aggregation_results = list(self.mongo_collection.aggregate(pipeline))
+        
+        # Transform the results to match the expected format
+        results = []
+        for result in aggregation_results:
+            # Skip null groups if needed
+            if result["_id"] is None and kwargs.get("skip_nulls", False):
+                continue
+                
+            # Create the group object
+            if isinstance(result["_id"], dict):
+                # Multi-field grouping
+                group_obj = result["_id"]
+            else:
+                # Single field grouping
+                group_obj = {group_by_fields[0]: result["_id"]}
+                
+            # Add the grouped objects
+            objects = result["objects"]
+            
+            # Remove MongoDB _id field from each object
+            for obj in objects:
+                if "_id" in obj:
+                    del obj["_id"]
+            
+            # Apply any field selection or transformations based on agg_map
+            if agg_map:
+                # Get first fields (fields to keep as single values)
+                first_fields = agg_map.get("first", [])
+                if first_fields:
+                    # These are already in the group_obj from the _id
+                    pass
+                
+                # Get list fields (fields to aggregate as lists)
+                list_fields = agg_map.get("list", [])
+                if list_fields:
+                    # Filter objects to only include specified fields
+                    objects = [{k: obj.get(k) for k in list_fields if k in obj} for obj in objects]
+                elif not list_fields and first_fields:
+                    # If list_fields is empty but first_fields is specified,
+                    # filter out first_fields from objects to avoid duplication
+                    objects = [{k: v for k, v in obj.items() if k not in first_fields} for obj in objects]
+            
+            # Add the objects to the group
+            group_obj[inlined_field] = objects
+            results.append(group_obj)
+            
+        return QueryResult(num_rows=len(results), rows=results)

linkml_store/api/stores/mongodb/mongodb_database.py

@@ -41,8 +41,9 @@ class MongoDBDatabase(Database):
         if self.handle:
             parsed_url = urlparse(self.handle)
             path_parts = parsed_url.path.lstrip("/").split("?")[0].split("/")
-            print(path_parts)
             db_name = path_parts[0] if path_parts else "default"
+            if not db_name:
+                db_name = self.alias
         else:
             db_name = "default"
         return db_name

linkml_store/api/stores/solr/solr_collection.py

@@ -62,12 +62,18 @@ class SolrCollection(Collection):
         return QueryResult(query=query, num_rows=num_rows, rows=rows)
 
     def query_facets(
-        self, where: Optional[Dict] = None, facet_columns: List[str] = None, facet_limit=DEFAULT_FACET_LIMIT, **kwargs
+        self,
+        where: Optional[Dict] = None,
+        facet_columns: List[str] = None,
+        facet_limit=DEFAULT_FACET_LIMIT,
+        facet_min_count: int = 1,
+        **kwargs,
     ) -> Dict[str, Dict[str, int]]:
         solr_query = self._build_solr_query(where)
         solr_query["facet"] = "true"
         solr_query["facet.field"] = facet_columns
         solr_query["facet.limit"] = facet_limit
+        solr_query["facet.mincount"] = facet_min_count
 
         logger.info(f"Querying Solr collection {self.alias} for facets with query: {solr_query}")